def channel_search(queue, channel_parameters, category, title_year, tecleado):
try:
search_results = []
exec "from channels import " + channel_parameters["channel"] + " as module"
mainlist = module.mainlist(Item(channel=channel_parameters["channel"]))
for item in mainlist:
if item.action != "search" or category and item.extra != category:
continue
for res_item in module.search(item.clone(), tecleado):
title = res_item.fulltitle
# If the release year is known, check if it matches the year found in the title
if title_year > 0:
year_match = re.search('\(.*(\d{4}).*\)', title)
if year_match and abs(int(year_match.group(1)) - title_year) > 1:
continue
# Clean up a bit the returned title to improve the fuzzy matching
title = re.sub(r'\(.*\)', '', title) # Anything within ()
title = re.sub(r'\[.*\]', '', title) # Anything within []
# Check if the found title fuzzy matches the searched one
if fuzz.token_sort_ratio(tecleado, title) > 85:
res_item.title = "[COLOR azure]" + res_item.title + "[/COLOR][COLOR orange] su [/COLOR][COLOR green]" + channel_parameters["title"] + "[/COLOR]"
search_results.append(res_item)
queue.put(search_results)
except:
logger.error("No se puede buscar en: " + channel_parameters["title"])
import traceback
logger.error(traceback.format_exc())
def agruparXcontenido(list_result_canal, categoria):
global channels_ID_name
dict_contenidos = {}
list_result = []
for i in list_result_canal:
# Formatear titulo
i.title = get_title(i)
# Eliminar tildes y otros caracteres especiales para la key
import unicodedata
try:
newKey = i.title.lower().strip().decode("UTF-8")
newKey = ''.join((c for c in unicodedata.normalize('NFD', newKey)
if unicodedata.category(c) != 'Mn'))
except:
newKey = i.title
found = False
for k in dict_contenidos.keys():
if fuzz.token_sort_ratio(newKey, k) > 85:
# Si el contenido ya estaba en el diccionario añadirlo a la lista de opciones...
dict_contenidos[k].append(i)
found = True
break
if not found: # ...sino añadirlo al diccionario
dict_contenidos[newKey] = [i]
# Añadimos el contenido encontrado en la lista list_result
for v in dict_contenidos.values():
title = v[0].title
if len(v) > 1:
# Eliminar de la lista de nombres de canales los q esten duplicados
canales_no_duplicados = []
for i in v:
if not i.channel in canales_no_duplicados:
canales_no_duplicados.append(channels_ID_name[i.channel])
if len(canales_no_duplicados) > 1:
canales = ', '.join([i for i in canales_no_duplicados[:-1]])
title += " (In %s e %s)" % (canales, canales_no_duplicados[-1])
else:
title += " (In %s)" % (', '.join(
[i for i in canales_no_duplicados]))
newItem = v[0].clone(channel="novedades",
title=title,
action="ver_canales",
sub_list=[i.tourl() for i in v],
extra=channels_ID_name)
else:
newItem = v[0].clone(title=title)
newItem.text_color = color3
list_result.append(newItem)
return sorted(list_result, key=lambda i: i.title.lower())
def do_channels_search(item):
logger.info("streamondemand.channels.biblioteca do_channels_search")
try:
title_year = int(item.extra[0:4])
except:
title_year = 0
mostra = item.extra[4:]
tecleado = urllib.quote_plus(mostra)
itemlist = []
import os
import glob
import imp
from lib.fuzzywuzzy import fuzz
import threading
import Queue
import time
master_exclude_data_file = os.path.join(config.get_runtime_path(), "resources", "sodsearch.txt")
logger.info("streamondemand.channels.buscador master_exclude_data_file=" + master_exclude_data_file)
channels_path = os.path.join(config.get_runtime_path(), "channels", '*.py')
logger.info("streamondemand.channels.buscador channels_path=" + channels_path)
excluir = ""
if os.path.exists(master_exclude_data_file):
logger.info("streamondemand.channels.buscador Encontrado fichero exclusiones")
fileexclude = open(master_exclude_data_file, "r")
excluir = fileexclude.read()
fileexclude.close()
else:
logger.info("streamondemand.channels.buscador No encontrado fichero exclusiones")
excluir = "seriesly\nbuscador\ntengourl\n__init__"
if config.is_xbmc():
show_dialog = True
try:
import xbmcgui
progreso = xbmcgui.DialogProgressBG()
progreso.create(NLS_Looking_For % mostra)
except:
show_dialog = False
def worker(infile, queue):
channel_result_itemlist = []
try:
basename_without_extension = os.path.basename(infile)[:-3]
# http://docs.python.org/library/imp.html?highlight=imp#module-imp
obj = imp.load_source(basename_without_extension, infile)
logger.info("streamondemand.channels.buscador cargado " + basename_without_extension + " de " + infile)
# item.url contains search type: serie, anime, etc...
channel_result_itemlist.extend(obj.search(Item(extra=item.url), tecleado))
for local_item in channel_result_itemlist:
local_item.title = " [COLOR azure] " + local_item.title + " [/COLOR] [COLOR orange]su[/COLOR] [COLOR green]" + basename_without_extension + "[/COLOR]"
local_item.viewmode = "list"
except:
import traceback
logger.error(traceback.format_exc())
queue.put(channel_result_itemlist)
channel_files = [infile for infile in glob.glob(channels_path) if os.path.basename(infile)[:-3] not in excluir]
result = Queue.Queue()
threads = [threading.Thread(target=worker, args=(infile, result)) for infile in channel_files]
start_time = int(time.time())
for t in threads:
t.daemon = True # NOTE: setting dameon to True allows the main thread to exit even if there are threads still running
t.start()
number_of_channels = len(channel_files)
completed_channels = 0
while completed_channels < number_of_channels:
delta_time = int(time.time()) - start_time
if len(itemlist) <= 0:
timeout = None # No result so far,lets the thread to continue working until a result is returned
elif delta_time >= TIMEOUT_TOTAL:
break # At least a result matching the searched title has been found, lets stop the search
else:
timeout = TIMEOUT_TOTAL - delta_time # Still time to gather other results
if show_dialog:
progreso.update(completed_channels * 100 / number_of_channels)
try:
result_itemlist = result.get(timeout=timeout)
completed_channels += 1
except:
# Expired timeout raise an exception
break
for item in result_itemlist:
title = item.fulltitle
# If the release year is known, check if it matches the year found in the title
if title_year > 0:
year_match = re.search('\(.*(\d{4}).*\)', title)
if year_match and abs(int(year_match.group(1)) - title_year) > 1:
continue
# Clean up a bit the returned title to improve the fuzzy matching
title = re.sub(r'\(.*\)', '', title) # Anything within ()
title = re.sub(r'\[.*\]', '', title) # Anything within []
# Check if the found title fuzzy matches the searched one
if fuzz.token_sort_ratio(mostra, title) > 85: itemlist.append(item)
if show_dialog:
progreso.close()
itemlist = sorted(itemlist, key=lambda item: item.fulltitle)
return itemlist
def do_channels_search(item):
logger.info("streamondemand-pureita-master.channels.biblioteca do_channels_search")
try:
title_year = int(item.extra[0:4])
except:
title_year = 0
mostra = item.extra[4:]
tecleado = urllib.quote_plus(mostra)
itemlist = []
channels_path = os.path.join(config.get_runtime_path(), "channels", "*.xml")
logger.info("streamondemand-pureita-master.channels.buscador channels_path=" + channels_path)
channel_language = config.get_setting("channel_language")
logger.info("streamondemand-pureita-master.channels.buscador channel_language=" + channel_language)
if channel_language == "":
channel_language = "all"
logger.info("streamondemand-pureita-master.channels.buscador channel_language=" + channel_language)
if config.is_xbmc():
show_dialog = True
try:
import xbmcgui
progreso = xbmcgui.DialogProgressBG()
progreso.create(NLS_Looking_For % mostra)
except:
show_dialog = False
def worker(infile, queue):
channel_result_itemlist = []
try:
basename_without_extension = os.path.basename(infile)[:-4]
# http://docs.python.org/library/imp.html?highlight=imp#module-imp
obj = imp.load_source(basename_without_extension, infile[:-4] + ".py")
logger.info(
"streamondemand-pureita-master.channels.buscador cargado "
+ basename_without_extension
+ " de "
+ infile
)
# item.url contains search type: serie, anime, etc...
channel_result_itemlist.extend(obj.search(Item(extra=item.url), tecleado))
for local_item in channel_result_itemlist:
local_item.title = (
" [COLOR azure] "
+ local_item.title
+ " [/COLOR] [COLOR orange]su[/COLOR] [COLOR green]"
+ basename_without_extension
+ "[/COLOR]"
)
local_item.viewmode = "list"
except:
import traceback
logger.error(traceback.format_exc())
queue.put(channel_result_itemlist)
channel_files = glob.glob(channels_path)
channel_files_tmp = []
for infile in channel_files:
basename_without_extension = os.path.basename(infile)[:-4]
channel_parameters = channeltools.get_channel_parameters(basename_without_extension)
# No busca si es un canal inactivo
if channel_parameters["active"] != "true":
continue
# No busca si es un canal excluido de la busqueda global
if channel_parameters["include_in_global_search"] != "true":
continue
# No busca si es un canal para adultos, y el modo adulto está desactivado
if channel_parameters["adult"] == "true" and config.get_setting("adult_mode") == "false":
continue
# No busca si el canal es en un idioma filtrado
if channel_language != "all" and channel_parameters["language"] != channel_language:
continue
channel_files_tmp.append(infile)
channel_files = channel_files_tmp
result = Queue.Queue()
threads = [threading.Thread(target=worker, args=(infile, result)) for infile in channel_files]
start_time = int(time.time())
for t in threads:
t.daemon = (
True
) # NOTE: setting dameon to True allows the main thread to exit even if there are threads still running
t.start()
number_of_channels = len(channel_files)
completed_channels = 0
while completed_channels < number_of_channels:
delta_time = int(time.time()) - start_time
if len(itemlist) <= 0:
timeout = None # No result so far,lets the thread to continue working until a result is returned
elif delta_time >= TIMEOUT_TOTAL:
break # At least a result matching the searched title has been found, lets stop the search
else:
timeout = TIMEOUT_TOTAL - delta_time # Still time to gather other results
if show_dialog:
progreso.update(completed_channels * 100 / number_of_channels)
try:
result_itemlist = result.get(timeout=timeout)
completed_channels += 1
except:
# Expired timeout raise an exception
break
for item in result_itemlist:
title = item.fulltitle
# If the release year is known, check if it matches the year found in the title
if title_year > 0:
year_match = re.search("\(.*(\d{4}).*\)", title)
if year_match and abs(int(year_match.group(1)) - title_year) > 1:
continue
# Clean up a bit the returned title to improve the fuzzy matching
title = re.sub(r"\(.*\)", "", title) # Anything within ()
title = re.sub(r"\[.*\]", "", title) # Anything within []
# Check if the found title fuzzy matches the searched one
if fuzz.token_sort_ratio(mostra, title) > 85:
itemlist.append(item)
if show_dialog:
progreso.close()
itemlist = sorted(itemlist, key=lambda item: item.fulltitle)
return itemlist
def do_channels_search(item):
logger.info("streamondemand.channels.biblioteca do_channels_search")
try:
title_year = int(item.extra[0:4])
except:
title_year = 0
mostra = item.extra[4:]
tecleado = urllib.quote_plus(mostra)
itemlist = []
import os
import glob
import imp
from lib.fuzzywuzzy import fuzz
import threading
import Queue
import time
master_exclude_data_file = os.path.join(config.get_runtime_path(),
"resources", "sodsearch.txt")
logger.info("streamondemand.channels.buscador master_exclude_data_file=" +
master_exclude_data_file)
channels_path = os.path.join(config.get_runtime_path(), "channels", '*.py')
logger.info("streamondemand.channels.buscador channels_path=" +
channels_path)
excluir = ""
if os.path.exists(master_exclude_data_file):
logger.info(
"streamondemand.channels.buscador Encontrado fichero exclusiones")
fileexclude = open(master_exclude_data_file, "r")
excluir = fileexclude.read()
fileexclude.close()
else:
logger.info(
"streamondemand.channels.buscador No encontrado fichero exclusiones"
)
excluir = "seriesly\nbuscador\ntengourl\n__init__"
if config.is_xbmc():
show_dialog = True
try:
import xbmcgui
progreso = xbmcgui.DialogProgressBG()
progreso.create(NLS_Looking_For % mostra)
except:
show_dialog = False
def worker(infile, queue):
channel_result_itemlist = []
try:
basename_without_extension = os.path.basename(infile)[:-3]
# http://docs.python.org/library/imp.html?highlight=imp#module-imp
obj = imp.load_source(basename_without_extension, infile)
logger.info("streamondemand.channels.buscador cargado " +
basename_without_extension + " de " + infile)
# item.url contains search type: serie, anime, etc...
channel_result_itemlist.extend(
obj.search(Item(extra=item.url), tecleado))
for local_item in channel_result_itemlist:
local_item.title = " [COLOR azure] " + local_item.title + " [/COLOR] [COLOR orange]su[/COLOR] [COLOR green]" + basename_without_extension + "[/COLOR]"
local_item.viewmode = "list"
except:
import traceback
logger.error(traceback.format_exc())
queue.put(channel_result_itemlist)
channel_files = [
infile for infile in glob.glob(channels_path)
if os.path.basename(infile)[:-3] not in excluir
]
result = Queue.Queue()
threads = [
threading.Thread(target=worker, args=(infile, result))
for infile in channel_files
]
start_time = int(time.time())
for t in threads:
t.daemon = True # NOTE: setting dameon to True allows the main thread to exit even if there are threads still running
t.start()
number_of_channels = len(channel_files)
completed_channels = 0
while completed_channels < number_of_channels:
delta_time = int(time.time()) - start_time
if len(itemlist) <= 0:
timeout = None # No result so far,lets the thread to continue working until a result is returned
elif delta_time >= TIMEOUT_TOTAL:
break # At least a result matching the searched title has been found, lets stop the search
else:
timeout = TIMEOUT_TOTAL - delta_time # Still time to gather other results
if show_dialog:
progreso.update(completed_channels * 100 / number_of_channels)
try:
result_itemlist = result.get(timeout=timeout)
completed_channels += 1
except:
# Expired timeout raise an exception
break
for item in result_itemlist:
title = item.fulltitle
# If the release year is known, check if it matches the year found in the title
if title_year > 0:
year_match = re.search('\(.*(\d{4}).*\)', title)
if year_match and abs(int(year_match.group(1)) -
title_year) > 1:
continue
# Clean up a bit the returned title to improve the fuzzy matching
title = re.sub(r'\(.*\)', '', title) # Anything within ()
title = re.sub(r'\[.*\]', '', title) # Anything within []
# Check if the found title fuzzy matches the searched one
if fuzz.token_sort_ratio(mostra, title) > 85: itemlist.append(item)
if show_dialog:
progreso.close()
itemlist = sorted(itemlist, key=lambda item: item.fulltitle)
return itemlist
def search_tor_book(books=None, mags=None):
if not (lazylibrarian.USE_TOR):
return
# rename this thread
threading.currentThread().name = "SEARCHTORBOOKS"
myDB = database.DBConnection()
searchlist = []
searchlist1 = []
if books is None:
# We are performing a backlog search
searchbooks = myDB.select(
'SELECT BookID, AuthorName, Bookname from books WHERE Status="Wanted"'
)
# Clear cache
if os.path.exists(".ProviderCache"):
for f in os.listdir(".ProviderCache"):
os.unlink("%s/%s" % (".ProviderCache", f))
# Clearing throttling timeouts
t = SimpleCache.ThrottlingProcessor()
t.lastRequestTime.clear()
else:
# The user has added a new book
searchbooks = []
if books != False:
for book in books:
searchbook = myDB.select(
'SELECT BookID, AuthorName, BookName from books WHERE BookID=? AND Status="Wanted"',
[book['bookid']])
for terms in searchbook:
searchbooks.append(terms)
for searchbook in searchbooks:
bookid = searchbook[0]
author = searchbook[1]
book = searchbook[2]
dic = {
'...': '',
'.': ' ',
' & ': ' ',
' = ': ' ',
'?': '',
'$': 's',
' + ': ' ',
'"': '',
',': '',
'*': '',
':': '',
';': ''
}
dicSearchFormatting = {'.': ' +', ' + ': ' '}
author = formatter.latinToAscii(formatter.replace_all(author, dic))
book = formatter.latinToAscii(formatter.replace_all(book, dic))
# TRY SEARCH TERM just using author name and book type
author = formatter.latinToAscii(
formatter.replace_all(author, dicSearchFormatting))
searchterm = author + ' ' + book # + ' ' + lazylibrarian.EBOOK_TYPE
searchterm = re.sub('[\.\-\/]', ' ', searchterm).encode('utf-8')
searchterm = re.sub(r'\(.*?\)', '', searchterm).encode('utf-8')
searchterm = re.sub(r"\s\s+", " ",
searchterm) # strip any double white space
searchlist.append({
"bookid": bookid,
"bookName": searchbook[2],
"authorName": searchbook[1],
"searchterm": searchterm.strip()
})
if not lazylibrarian.KAT:
logger.info('No download method is set, use SABnzbd or blackhole')
counter = 0
for book in searchlist:
#print book.keys()
resultlist = providers.IterateOverTorrentSites(book, 'book')
#if you can't find teh book specifically, you might find under general search
if not resultlist:
logger.info(
"Searching for type book failed to find any books...moving to general search"
)
resultlist = providers.IterateOverTorrentSites(book, 'general')
if not resultlist:
logger.debug("Adding book %s to queue." % book['searchterm'])
else:
dictrepl = {
'...': '',
'.': ' ',
' & ': ' ',
' = ': ' ',
'?': '',
'$': 's',
' + ': ' ',
'"': '',
',': '',
'*': '',
'(': '',
')': '',
'[': '',
']': '',
'#': '',
'0': '',
'1': '',
'2': '',
'3': '',
'4': '',
'5': '',
'6': '',
'7': '',
'8': '',
'9': '',
'\'': '',
':': '',
'!': '',
'-': '',
'\s\s': ' ',
' the ': ' ',
' a ': ' ',
' and ': ' ',
' to ': ' ',
' of ': ' ',
' for ': ' ',
' my ': ' ',
' in ': ' ',
' at ': ' ',
' with ': ' '
}
logger.debug(u'searchterm %s' % book['searchterm'])
addedCounter = 0
for tor in resultlist:
tor_Title = formatter.latinToAscii(
formatter.replace_all(
str(tor['tor_title']).lower(), dictrepl)).strip()
tor_Title = re.sub(r"\s\s+", " ",
tor_Title) #remove extra whitespace
logger.debug(u'torName %s' % tor_Title)
match_ratio = int(lazylibrarian.MATCH_RATIO)
tor_Title_match = fuzz.token_sort_ratio(
book['searchterm'].lower(), tor_Title)
logger.debug("Torrent Title Match %: " + str(tor_Title_match))
if (tor_Title_match > match_ratio):
logger.info(u'Found Torrent: %s' % tor['tor_title'])
addedCounter = addedCounter + 1
bookid = book['bookid']
tor_Title = (book["authorName"] + ' - ' +
book['bookName'] + ' LL.(' + book['bookid'] +
')').strip()
tor_url = tor['tor_url']
tor_prov = tor['tor_prov']
tor_size_temp = tor[
'tor_size'] #Need to cater for when this is NONE (Issue 35)
if tor_size_temp is None:
tor_size_temp = 1000
tor_size = str(round(float(tor_size_temp) / 1048576,
2)) + ' MB'
controlValueDict = {"NZBurl": tor_url}
newValueDict = {
"NZBprov": tor_prov,
"BookID": bookid,
"NZBsize": tor_size,
"NZBtitle": tor_Title,
"Status": "Skipped"
}
myDB.upsert("wanted", newValueDict, controlValueDict)
snatchedbooks = myDB.action(
'SELECT * from books WHERE BookID=? and Status="Snatched"',
[bookid]).fetchone()
if not snatchedbooks:
snatch = DownloadMethod(bookid, tor_prov, tor_Title,
tor_url)
notifiers.notify_snatch(tor_Title + ' at ' +
formatter.now())
break
if addedCounter == 0:
logger.info("No torrent's found for " +
(book["authorName"] + ' ' +
book['bookName']).strip() +
". Adding book to queue.")
counter = counter + 1
# if not books or books==False:
# snatched = searchmag.searchmagazines(mags)
# for items in snatched:
# snatch = DownloadMethod(items['bookid'], items['tor_prov'], items['tor_title'], items['tor_url'])
# notifiers.notify_snatch(items['tor_title']+' at '+formatter.now())
logger.info("Search for Wanted items complete")
def search_tor_book(books=None, mags=None):
if not(lazylibrarian.USE_TOR):
return
# rename this thread
threading.currentThread().name = "SEARCHTORBOOKS"
myDB = database.DBConnection()
searchlist = []
searchlist1 = []
if books is None:
# We are performing a backlog search
searchbooks = myDB.select('SELECT BookID, AuthorName, Bookname from books WHERE Status="Wanted"')
# Clear cache
if os.path.exists(".ProviderCache"):
for f in os.listdir(".ProviderCache"):
os.unlink("%s/%s" % (".ProviderCache", f))
# Clearing throttling timeouts
t = SimpleCache.ThrottlingProcessor()
t.lastRequestTime.clear()
else:
# The user has added a new book
searchbooks = []
if books != False:
for book in books:
searchbook = myDB.select('SELECT BookID, AuthorName, BookName from books WHERE BookID=? AND Status="Wanted"', [book['bookid']])
for terms in searchbook:
searchbooks.append(terms)
for searchbook in searchbooks:
bookid = searchbook[0]
author = searchbook[1]
book = searchbook[2]
dic = {'...':'', '.':' ', ' & ':' ', ' = ': ' ', '?':'', '$':'s', ' + ':' ', '"':'', ',':'', '*':'', ':':'', ';':''}
dicSearchFormatting = {'.':' +', ' + ':' '}
author = formatter.latinToAscii(formatter.replace_all(author, dic))
book = formatter.latinToAscii(formatter.replace_all(book, dic))
# TRY SEARCH TERM just using author name and book type
author = formatter.latinToAscii(formatter.replace_all(author, dicSearchFormatting))
searchterm = author + ' ' + book # + ' ' + lazylibrarian.EBOOK_TYPE
searchterm = re.sub('[\.\-\/]', ' ', searchterm).encode('utf-8')
searchterm = re.sub(r'\(.*?\)', '', searchterm).encode('utf-8')
searchterm = re.sub(r"\s\s+" , " ", searchterm) # strip any double white space
searchlist.append({"bookid": bookid, "bookName":searchbook[2], "authorName":searchbook[1], "searchterm": searchterm.strip()})
if not lazylibrarian.KAT:
logger.info('No download method is set, use SABnzbd or blackhole')
counter = 0
for book in searchlist:
#print book.keys()
resultlist = providers.IterateOverTorrentSites(book,'book')
#if you can't find teh book specifically, you might find under general search
if not resultlist:
logger.info("Searching for type book failed to find any books...moving to general search")
resultlist = providers.IterateOverTorrentSites(book,'general')
if not resultlist:
logger.debug("No result found, Adding book %s to queue " % book['searchterm'])
else:
dictrepl = {'...':'', '.':' ', ' & ':' ', ' = ': ' ', '?':'', '$':'s', ' + ':' ', '"':'', ',':'', '*':'', '(':'', ')':'', '[':'', ']':'', '#':'', '0':'', '1':'', '2':'', '3':'', '4':'', '5':'', '6':'', '7':'', '8':'' , '9':'', '\'':'', ':':'', '!':'', '-':'', '\s\s':' ', ' the ':' ', ' a ':' ', ' and ':' ', ' to ':' ', ' of ':' ', ' for ':' ', ' my ':' ', ' in ':' ', ' at ':' ', ' with ':' ' }
logger.debug(u'searchterm %s' % book['searchterm'])
addedCounter = 0
for tor in resultlist:
tor_Title = formatter.latinToAscii(formatter.replace_all(str(tor['tor_title']).lower(), dictrepl)).strip()
tor_Title = re.sub(r"\s\s+" , " ", tor_Title) #remove extra whitespace
logger.debug(u'torName %s' % tor_Title)
match_ratio = int(lazylibrarian.MATCH_RATIO)
tor_Title_match = fuzz.token_sort_ratio(book['searchterm'].lower(), tor_Title)
logger.debug("Torrent Title Match %: " + str(tor_Title_match))
if (tor_Title_match > match_ratio):
logger.info(u'Found Torrent: %s' % tor['tor_title'])
addedCounter = addedCounter + 1
bookid = book['bookid']
tor_Title = (book["authorName"] + ' - ' + book['bookName'] + ' LL.(' + book['bookid'] + ')').strip()
tor_url = tor['tor_url']
tor_prov = tor['tor_prov']
tor_size_temp = tor['tor_size'] #Need to cater for when this is NONE (Issue 35)
if tor_size_temp is None:
tor_size_temp = 1000
tor_size = str(round(float(tor_size_temp) / 1048576,2))+' MB'
controlValueDict = {"NZBurl": tor_url}
newValueDict = {
"NZBprov": tor_prov,
"BookID": bookid,
"NZBsize": tor_size,
"NZBtitle": tor_Title,
"Status": "Skipped"
}
myDB.upsert("wanted", newValueDict, controlValueDict)
snatchedbooks = myDB.action('SELECT * from books WHERE BookID=? and Status="Snatched"', [bookid]).fetchone()
if not snatchedbooks:
snatch = DownloadMethod(bookid, tor_prov, tor_Title, tor_url)
notifiers.notify_snatch(tor_Title+' at '+formatter.now())
break;
if addedCounter == 0:
logger.info("No torrent's found for " + (book["authorName"] + ' ' + book['bookName']).strip() + ". Adding book to queue.")
counter = counter + 1
# if not books or books==False:
# snatched = searchmag.searchmagazines(mags)
# for items in snatched:
# snatch = DownloadMethod(items['bookid'], items['tor_prov'], items['tor_title'], items['tor_url'])
# notifiers.notify_snatch(items['tor_title']+' at '+formatter.now())
logger.info("Search for Wanted items complete")
def do_channels_search(item):
logger.info("streamondemand-pureita.channels.biblioteca do_channels_search")
try:
title_year = int(item.extra[0:4])
except:
title_year = 0
mostra = item.extra[4:]
tecleado = urllib.quote_plus(mostra)
itemlist = []
channels_path = os.path.join(config.get_runtime_path(), "channels", '*.xml')
logger.info("streamondemand-pureita.channels.buscador channels_path=" + channels_path)
channel_language = config.get_setting("channel_language")
logger.info("streamondemand-pureita.channels.buscador channel_language=" + channel_language)
if channel_language == "":
channel_language = "all"
logger.info("streamondemand-pureita.channels.buscador channel_language=" + channel_language)
if config.is_xbmc():
show_dialog = True
try:
import xbmcgui
progreso = xbmcgui.DialogProgressBG()
progreso.create(NLS_Looking_For % mostra)
except:
show_dialog = False
def worker(infile, queue):
channel_result_itemlist = []
try:
basename_without_extension = os.path.basename(infile)[:-4]
# http://docs.python.org/library/imp.html?highlight=imp#module-imp
obj = imp.load_source(basename_without_extension, infile[:-4]+".py")
logger.info("streamondemand-pureita.channels.buscador cargado " + basename_without_extension + " de " + infile)
# item.url contains search type: serie, anime, etc...
channel_result_itemlist.extend(obj.search(Item(extra=item.url), tecleado))
for local_item in channel_result_itemlist:
local_item.title = " [COLOR azure] " + local_item.title + " [/COLOR] [COLOR orange]su[/COLOR] [COLOR orange]" + basename_without_extension + "[/COLOR]"
local_item.viewmode = "list"
except:
import traceback
logger.error(traceback.format_exc())
queue.put(channel_result_itemlist)
channel_files = glob.glob(channels_path)
channel_files_tmp = []
for infile in channel_files:
basename_without_extension = os.path.basename(infile)[:-4]
channel_parameters = channeltools.get_channel_parameters(basename_without_extension)
# Non cercare se il canale e inattivo
if channel_parameters["active"] != "true":
continue
# Non cercare se un canale e escluso dalla ricerca globale
if channel_parameters["include_in_global_search"] != "true":
continue
# Non cercare se un canale e per adulti e la modalita adulta disabilitata
if channel_parameters["adult"] == "true" and config.get_setting("adult_mode") == "false":
continue
# Non cercare se un canale ha il filtro lingua
if channel_language != "all" and channel_parameters["language"] != channel_language:
continue
channel_files_tmp.append(infile)
channel_files = channel_files_tmp
result = Queue.Queue()
threads = [threading.Thread(target=worker, args=(infile, result)) for infile in channel_files]
start_time = int(time.time())
for t in threads:
t.daemon = True # NOTE: setting dameon to True allows the main thread to exit even if there are threads still running
t.start()
number_of_channels = len(channel_files)
completed_channels = 0
while completed_channels < number_of_channels:
delta_time = int(time.time()) - start_time
if len(itemlist) <= 0:
timeout = None # No result so far,lets the thread to continue working until a result is returned
elif delta_time >= TIMEOUT_TOTAL:
break # At least a result matching the searched title has been found, lets stop the search
else:
timeout = TIMEOUT_TOTAL - delta_time # Still time to gather other results
if show_dialog:
progreso.update(completed_channels * 100 / number_of_channels)
try:
result_itemlist = result.get(timeout=timeout)
completed_channels += 1
except:
# Expired timeout raise an exception
break
for item in result_itemlist:
title = item.fulltitle
# If the release year is known, check if it matches the year found in the title
if title_year > 0:
year_match = re.search('\(.*(\d{4}).*\)', title)
if year_match and abs(int(year_match.group(1)) - title_year) > 1:
continue
# Clean up a bit the returned title to improve the fuzzy matching
title = re.sub(r'\(.*\)', '', title) # Anything within ()
title = re.sub(r'\[.*\]', '', title) # Anything within []
# Check if the found title fuzzy matches the searched one
if fuzz.token_sort_ratio(mostra, title) > 85: itemlist.append(item)
if show_dialog:
progreso.close()
itemlist = sorted(itemlist, key=lambda item: item.fulltitle)
return itemlist
def syncCalibreList(col_read=None, col_toread=None, userid=None):
""" Get the lazylibrarian bookid for each read/toread calibre book so we can map our id to theirs,
and sync current/supplied user's read/toread or supplied read/toread columns to calibre database.
Return message giving totals """
myDB = database.DBConnection()
if not userid:
cookie = cherrypy.request.cookie
if cookie and 'll_uid' in cookie.keys():
userid = cookie['ll_uid'].value
if userid:
res = myDB.match(
'SELECT UserName,ToRead,HaveRead,CalibreRead,CalibreToRead,Perms from users where UserID=?',
(userid, ))
if res:
username = res['UserName']
if not col_read:
col_read = res['CalibreRead']
if not col_toread:
col_toread = res['CalibreToRead']
toreadlist = getList(res['ToRead'])
readlist = getList(res['HaveRead'])
# suppress duplicates (just in case)
toreadlist = list(set(toreadlist))
readlist = list(set(readlist))
else:
return "Error: Unable to get user column settings for %s" % userid
if not userid:
return "Error: Unable to find current userid"
if not col_read and not col_toread:
return "User %s has no calibre columns set" % username
# check user columns exist in calibre and create if not
res = calibredb('custom_columns')
columns = res[0].split('\n')
custom_columns = []
for column in columns:
if column:
custom_columns.append(column.split(' (')[0])
if col_read not in custom_columns:
added = calibredb('add_custom_column', [col_read, col_read, 'bool'])
if "column created" not in added[0]:
return added
if col_toread not in custom_columns:
added = calibredb('add_custom_column',
[col_toread, col_toread, 'bool'])
if "column created" not in added[0]:
return added
nomatch = 0
readcol = ''
toreadcol = ''
map_ctol = {}
map_ltoc = {}
if col_read:
readcol = '*' + col_read
if col_toread:
toreadcol = '*' + col_toread
calibre_list = calibreList(col_read, col_toread)
if not isinstance(calibre_list, list):
# got an error message from calibredb
return '"%s"' % calibre_list
for item in calibre_list:
if toreadcol and toreadcol in item or readcol and readcol in item:
authorname, authorid, added = addAuthorNameToDB(item['authors'],
refresh=False,
addbooks=False)
if authorname:
if authorname != item['authors']:
logger.debug(
"Changed authorname for [%s] from [%s] to [%s]" %
(item['title'], item['authors'], authorname))
item['authors'] = authorname
bookid = find_book_in_db(authorname, item['title'])
if not bookid:
searchterm = "%s <ll> %s" % (item['title'], authorname)
results = search_for(unaccented(searchterm))
if results:
result = results[0]
if result['author_fuzz'] > lazylibrarian.CONFIG['MATCH_RATIO'] \
and result['book_fuzz'] > lazylibrarian.CONFIG['MATCH_RATIO']:
logger.debug(
"Found (%s%% %s%%) %s: %s" %
(result['author_fuzz'], result['book_fuzz'],
result['authorname'], result['bookname']))
bookid = result['bookid']
import_book(bookid)
if bookid:
# NOTE: calibre bookid is always an integer, lazylibrarian bookid is a string
# (goodreads could be used as an int, but googlebooks can't as it's alphanumeric)
# so convert all dict items to strings for ease of matching.
map_ctol[str(item['id'])] = str(bookid)
map_ltoc[str(bookid)] = str(item['id'])
else:
logger.warn(
'Calibre Book [%s] by [%s] is not in lazylibrarian database'
% (item['title'], authorname))
nomatch += 1
else:
logger.warn(
'Calibre Author [%s] not matched in lazylibrarian database'
% (item['authors']))
nomatch += 1
# Now check current users lazylibrarian read/toread against the calibre library, warn about missing ones
# which might be books calibre doesn't have, or might be minor differences in author or title
for idlist in [("Read", readlist), ("To_Read", toreadlist)]:
booklist = idlist[1]
for bookid in booklist:
cmd = "SELECT AuthorID,BookName from books where BookID=?"
book = myDB.match(cmd, (bookid, ))
if not book:
logger.error('Error finding bookid %s' % bookid)
else:
cmd = "SELECT AuthorName from authors where AuthorID=?"
author = myDB.match(cmd, (book['AuthorID'], ))
if not author:
logger.error('Error finding authorid %s' %
book['AuthorID'])
else:
match = False
for item in calibre_list:
if item['authors'] == author['AuthorName'] and item[
'title'] == book['BookName']:
logger.debug("Exact match for %s [%s]" %
(idlist[0], book['BookName']))
map_ctol[str(item['id'])] = str(bookid)
map_ltoc[str(bookid)] = str(item['id'])
match = True
break
if not match:
high = 0
highname = ''
highid = ''
for item in calibre_list:
if item['authors'] == author['AuthorName']:
n = fuzz.token_sort_ratio(
item['title'], book['BookName'])
if n > high:
high = n
highname = item['title']
highid = item['id']
if high > 95:
logger.debug(
"Found ratio match %s%% [%s] for %s [%s]" %
(high, highname, idlist[0], book['BookName']))
map_ctol[str(highid)] = str(bookid)
map_ltoc[str(bookid)] = str(highid)
match = True
if not match:
logger.warn(
"No match for %s %s by %s in calibre database, closest match %s%% [%s]"
% (idlist[0], book['BookName'],
author['AuthorName'], high, highname))
nomatch += 1
logger.debug("BookID mapping complete, %s match %s, nomatch %s" %
(username, len(map_ctol), nomatch))
# now sync the lists
if userid:
last_read = []
last_toread = []
calibre_read = []
calibre_toread = []
cmd = 'select SyncList from sync where UserID=? and Label=?'
res = myDB.match(cmd, (userid, col_read))
if res:
last_read = getList(res['SyncList'])
res = myDB.match(cmd, (userid, col_toread))
if res:
last_toread = getList(res['SyncList'])
for item in calibre_list:
if toreadcol and toreadcol in item and item[
toreadcol]: # only if True
if str(item['id']) in map_ctol:
calibre_toread.append(map_ctol[str(item['id'])])
else:
logger.warn(
"Calibre to_read book %s:%s has no lazylibrarian bookid"
% (item['authors'], item['title']))
if readcol and readcol in item and item[readcol]: # only if True
if str(item['id']) in map_ctol:
calibre_read.append(map_ctol[str(item['id'])])
else:
logger.warn(
"Calibre read book %s:%s has no lazylibrarian bookid" %
(item['authors'], item['title']))
logger.debug("Found %s calibre read, %s calibre toread" %
(len(calibre_read), len(calibre_toread)))
logger.debug("Found %s lazylib read, %s lazylib toread" %
(len(readlist), len(toreadlist)))
added_to_ll_toread = list(set(toreadlist) - set(last_toread))
removed_from_ll_toread = list(set(last_toread) - set(toreadlist))
added_to_ll_read = list(set(readlist) - set(last_read))
removed_from_ll_read = list(set(last_read) - set(readlist))
logger.debug("lazylibrarian changes to copy to calibre: %s %s %s %s" %
(len(added_to_ll_toread), len(removed_from_ll_toread),
len(added_to_ll_read), len(removed_from_ll_read)))
added_to_calibre_toread = list(set(calibre_toread) - set(last_toread))
removed_from_calibre_toread = list(
set(last_toread) - set(calibre_toread))
added_to_calibre_read = list(set(calibre_read) - set(last_read))
removed_from_calibre_read = list(set(last_read) - set(calibre_read))
logger.debug(
"calibre changes to copy to lazylibrarian: %s %s %s %s" %
(len(added_to_calibre_toread), len(removed_from_calibre_toread),
len(added_to_calibre_read), len(removed_from_calibre_read)))
calibre_changes = 0
for item in added_to_calibre_read:
if item not in readlist:
readlist.append(item)
logger.debug("Lazylibrarian marked %s as read" % item)
calibre_changes += 1
for item in added_to_calibre_toread:
if item not in toreadlist:
toreadlist.append(item)
logger.debug("Lazylibrarian marked %s as to_read" % item)
calibre_changes += 1
for item in removed_from_calibre_read:
if item in readlist:
readlist.remove(item)
logger.debug("Lazylibrarian removed %s from read" % item)
calibre_changes += 1
for item in removed_from_calibre_toread:
if item in toreadlist:
toreadlist.remove(item)
logger.debug("Lazylibrarian removed %s from to_read" % item)
calibre_changes += 1
if calibre_changes:
myDB.action('UPDATE users SET ToRead=?,HaveRead=? WHERE UserID=?',
(', '.join(toreadlist), ', '.join(readlist), userid))
ll_changes = 0
for item in added_to_ll_toread:
if item in map_ltoc:
res, err, rc = calibredb('set_custom',
[col_toread, map_ltoc[item], 'true'],
[])
if rc:
msg = "calibredb set_custom error: "
if err:
logger.error(msg + err)
elif res:
logger.error(msg + res)
else:
logger.error(msg + str(rc))
else:
ll_changes += 1
else:
logger.warn("Unable to set calibre %s true for %s" %
(col_toread, item))
for item in removed_from_ll_toread:
if item in map_ltoc:
res, err, rc = calibredb('set_custom',
[col_toread, map_ltoc[item], ''], [])
if rc:
msg = "calibredb set_custom error: "
if err:
logger.error(msg + err)
elif res:
logger.error(msg + res)
else:
logger.error(msg + str(rc))
else:
ll_changes += 1
else:
logger.warn("Unable to clear calibre %s for %s" %
(col_toread, item))
for item in added_to_ll_read:
if item in map_ltoc:
res, err, rc = calibredb('set_custom',
[col_read, map_ltoc[item], 'true'],
[])
if rc:
msg = "calibredb set_custom error: "
if err:
logger.error(msg + err)
elif res:
logger.error(msg + res)
else:
logger.error(msg + str(rc))
else:
ll_changes += 1
else:
logger.warn("Unable to set calibre %s true for %s" %
(col_read, item))
for item in removed_from_ll_read:
if item in map_ltoc:
res, err, rc = calibredb('set_custom',
[col_read, map_ltoc[item], ''], [])
if rc:
msg = "calibredb set_custom error: "
if err:
logger.error(msg + err)
elif res:
logger.error(msg + res)
else:
logger.error(msg + str(rc))
else:
ll_changes += 1
else:
logger.warn("Unable to clear calibre %s for %s" %
(col_read, item))
# store current sync list as comparison for next sync
controlValueDict = {"UserID": userid, "Label": col_read}
newValueDict = {
"Date": str(time.time()),
"Synclist": ', '.join(readlist)
}
myDB.upsert("sync", newValueDict, controlValueDict)
controlValueDict = {"UserID": userid, "Label": col_toread}
newValueDict = {
"Date": str(time.time()),
"Synclist": ', '.join(toreadlist)
}
myDB.upsert("sync", newValueDict, controlValueDict)
msg = "%s sync updated: %s calibre, %s lazylibrarian" % (
username, ll_changes, calibre_changes)
return msg
if not resultlist:
logger.debug("Adding book %s to queue." % book['searchterm'])
else:
dictrepl = {'...': '', '.': ' ', ' & ': ' ', ' = ': ' ', '?': '', '$': 's', ' + ': ' ', '"': '', ',': '', '*': '', '(': '', ')': '', '[': '', ']': '', '#': '', '0': '', '1': '', '2': '', '3': '', '4': '', '5': '', '6': '', '7': '', '8': '', '9': '', '\'': '', ':': '', '!': '', '-': '', '\s\s': ' ', ' the ': ' ', ' a ': ' ', ' and ': ' ', ' to ': ' ', ' of ': ' ', ' for ': ' ', ' my ': ' ', ' in ': ' ', ' at ': ' ', ' with ': ' '}
logger.debug(u'searchterm %s' % book['searchterm'])
addedCounter = 0
for tor in resultlist:
tor_Title = formatter.latinToAscii(formatter.replace_all(str(tor['tor_title']).lower(), dictrepl)).strip()
tor_Title = re.sub(r"\s\s+", " ", tor_Title) # remove extra whitespace
logger.debug(u'torName %s' % tor_Title)
match_ratio = int(lazylibrarian.MATCH_RATIO)
tor_Title_match = fuzz.token_sort_ratio(book['searchterm'].lower(), tor_Title)
logger.debug("Torrent Title Match %: " + str(tor_Title_match))
if (tor_Title_match > match_ratio):
logger.debug(u'Found Torrent: %s' % tor['tor_title'])
addedCounter = addedCounter + 1
bookid = book['bookid']
tor_Title = (book["authorName"] + ' - ' + book['bookName'] + ' LL.(' + book['bookid'] + ')').strip()
tor_url = tor['tor_url']
tor_prov = tor['tor_prov']
tor_size_temp = tor['tor_size'] # Need to cater for when this is NONE (Issue 35)
if tor_size_temp is None:
tor_size_temp = 1000
tor_size = str(round(float(tor_size_temp) / 1048576, 2)) + ' MB'
def search_nzb_book(books=None, mags=None):
if not (lazylibrarian.USE_NZB):
return
# rename this thread
threading.currentThread().name = "SEARCHNZBBOOKS"
myDB = database.DBConnection()
searchlist = []
searchlist1 = []
if books is None:
# We are performing a backlog search
searchbooks = myDB.select('SELECT BookID, AuthorName, Bookname from books WHERE Status="Wanted"')
# Clear cache
if os.path.exists(".ProviderCache"):
for f in os.listdir(".ProviderCache"):
os.unlink("%s/%s" % (".ProviderCache", f))
# Clearing throttling timeouts
t = SimpleCache.ThrottlingProcessor()
t.lastRequestTime.clear()
else:
# The user has added a new book
searchbooks = []
if books != False:
for book in books:
searchbook = myDB.select(
'SELECT BookID, AuthorName, BookName from books WHERE BookID=? AND Status="Wanted"',
[book["bookid"]],
)
for terms in searchbook:
searchbooks.append(terms)
for searchbook in searchbooks:
bookid = searchbook[0]
author = searchbook[1]
book = searchbook[2]
dic = {
"...": "",
".": " ",
" & ": " ",
" = ": " ",
"?": "",
"$": "s",
" + ": " ",
'"': "",
",": "",
"*": "",
":": "",
";": "",
}
dicSearchFormatting = {".": " +", " + ": " "}
author = formatter.latinToAscii(formatter.replace_all(author, dic))
book = formatter.latinToAscii(formatter.replace_all(book, dic))
# TRY SEARCH TERM just using author name and book type
author = formatter.latinToAscii(formatter.replace_all(author, dicSearchFormatting))
searchterm = author + " " + book # + ' ' + lazylibrarian.EBOOK_TYPE
searchterm = re.sub("[\.\-\/]", " ", searchterm).encode("utf-8")
searchterm = re.sub(r"\(.*?\)", "", searchterm).encode("utf-8")
searchterm = re.sub(r"\s\s+", " ", searchterm) # strip any double white space
searchlist.append(
{"bookid": bookid, "bookName": searchbook[2], "authorName": searchbook[1], "searchterm": searchterm.strip()}
)
if not lazylibrarian.SAB_HOST and not lazylibrarian.NZB_DOWNLOADER_BLACKHOLE:
logger.info("No download method is set, use SABnzbd or blackhole")
# TODO - Move the newznab test to providers.py
if not lazylibrarian.NEWZNAB and not lazylibrarian.NEWZNAB2 and not lazylibrarian.USENETCRAWLER:
logger.info("No providers are set. try use NEWZNAB.")
counter = 0
for book in searchlist:
# print book.keys()
resultlist = providers.IterateOverNewzNabSites(book, "book")
# if you can't find teh book specifically, you might find under general search
if not resultlist:
logger.info("Searching for type book failed to find any books...moving to general search")
resultlist = providers.IterateOverNewzNabSites(book, "general")
if not resultlist:
logger.debug("Adding book %s to queue." % book["searchterm"])
else:
dictrepl = {
"...": "",
".": " ",
" & ": " ",
" = ": " ",
"?": "",
"$": "s",
" + ": " ",
'"': "",
",": "",
"*": "",
"(": "",
")": "",
"[": "",
"]": "",
"#": "",
"0": "",
"1": "",
"2": "",
"3": "",
"4": "",
"5": "",
"6": "",
"7": "",
"8": "",
"9": "",
"'": "",
":": "",
"!": "",
"-": "",
"\s\s": " ",
" the ": " ",
" a ": " ",
" and ": " ",
" to ": " ",
" of ": " ",
" for ": " ",
" my ": " ",
" in ": " ",
" at ": " ",
" with ": " ",
}
logger.debug(u"searchterm %s" % book["searchterm"])
addedCounter = 0
for nzb in resultlist:
nzbTitle = formatter.latinToAscii(formatter.replace_all(str(nzb["nzbtitle"]).lower(), dictrepl)).strip()
nzbTitle = re.sub(r"\s\s+", " ", nzbTitle) # remove extra whitespace
logger.debug(u"nzbName %s" % nzbTitle)
match_ratio = int(lazylibrarian.MATCH_RATIO)
nzbTitle_match = fuzz.token_sort_ratio(book["searchterm"].lower(), nzbTitle)
logger.debug("NZB Title Match %: " + str(nzbTitle_match))
if nzbTitle_match > match_ratio:
logger.info(u"Found NZB: %s" % nzb["nzbtitle"])
addedCounter = addedCounter + 1
bookid = book["bookid"]
nzbTitle = (book["authorName"] + " - " + book["bookName"] + " LL.(" + book["bookid"] + ")").strip()
nzburl = nzb["nzburl"]
nzbprov = nzb["nzbprov"]
nzbdate_temp = nzb["nzbdate"]
nzbsize_temp = nzb["nzbsize"] # Need to cater for when this is NONE (Issue 35)
if nzbsize_temp is None:
nzbsize_temp = 1000
nzbsize = str(round(float(nzbsize_temp) / 1048576, 2)) + " MB"
nzbdate = formatter.nzbdate2format(nzbdate_temp)
controlValueDict = {"NZBurl": nzburl}
newValueDict = {
"NZBprov": nzbprov,
"BookID": bookid,
"NZBdate": nzbdate,
"NZBsize": nzbsize,
"NZBtitle": nzbTitle,
"Status": "Skipped",
}
myDB.upsert("wanted", newValueDict, controlValueDict)
snatchedbooks = myDB.action(
'SELECT * from books WHERE BookID=? and Status="Snatched"', [bookid]
).fetchone()
if not snatchedbooks:
snatch = DownloadMethod(bookid, nzbprov, nzbTitle, nzburl)
notifiers.notify_snatch(nzbTitle + " at " + formatter.now())
break
if addedCounter == 0:
logger.info(
"No nzb's found for "
+ (book["authorName"] + " " + book["bookName"]).strip()
+ ". Adding book to queue."
)
counter = counter + 1
if not books or books == False:
snatched = searchmag.searchmagazines(mags)
for items in snatched:
snatch = DownloadMethod(items["bookid"], items["nzbprov"], items["nzbtitle"], items["nzburl"])
notifiers.notify_snatch(items["nzbtitle"] + " at " + formatter.now())
logger.info("Search for Wanted items complete")
def do_channels_search(item):
logger.info("streamondemand.channels.biblioteca do_channels_search")
try:
title_year = int(item.extra[0:4])
except Exception:
title_year = 0
mostra = item.extra[4:]
tecleado = urllib.quote_plus(mostra)
itemlist = []
import os
import glob
import imp
from lib.fuzzywuzzy import fuzz
import threading
import Queue
master_exclude_data_file = os.path.join(config.get_runtime_path(), "resources", "sodsearch.txt")
logger.info("streamondemand.channels.buscador master_exclude_data_file=" + master_exclude_data_file)
channels_path = os.path.join(config.get_runtime_path(), "channels", '*.py')
logger.info("streamondemand.channels.buscador channels_path=" + channels_path)
excluir = ""
if os.path.exists(master_exclude_data_file):
logger.info("streamondemand.channels.buscador Encontrado fichero exclusiones")
fileexclude = open(master_exclude_data_file, "r")
excluir = fileexclude.read()
fileexclude.close()
else:
logger.info("streamondemand.channels.buscador No encontrado fichero exclusiones")
excluir = "seriesly\nbuscador\ntengourl\n__init__"
if config.is_xbmc():
show_dialog = True
try:
import xbmcgui
progreso = xbmcgui.DialogProgressBG()
progreso.create(NLS_Looking_For % mostra)
except:
show_dialog = False
def worker(infile, queue):
channel_result_itemlist = []
try:
basename_without_extension = os.path.basename(infile)[:-3]
# http://docs.python.org/library/imp.html?highlight=imp#module-imp
obj = imp.load_source(basename_without_extension, infile)
logger.info("streamondemand.channels.buscador cargado " + basename_without_extension + " de " + infile)
# item.url contains search type: serie, anime, etc...
channel_result_itemlist.extend(obj.search(Item(extra=item.url), tecleado))
for local_item in channel_result_itemlist:
local_item.title = " [COLOR azure] " + local_item.title + " [/COLOR] [COLOR orange]su[/COLOR] [COLOR green]" + basename_without_extension + "[/COLOR]"
local_item.viewmode = "list"
except:
import traceback
logger.error(traceback.format_exc())
queue.put(channel_result_itemlist)
channel_files = [infile for infile in glob.glob(channels_path) if os.path.basename(infile)[:-3] not in excluir]
result = Queue.Queue()
threads = [threading.Thread(target=worker, args=(infile, result)) for infile in channel_files]
for t in threads:
t.start()
number_of_channels = len(channel_files)
local_itemlist = []
for index, t in enumerate(threads):
percentage = index * 100 / number_of_channels
if show_dialog:
progreso.update(percentage, NLS_Looking_For % mostra)
t.join()
local_itemlist.extend(result.get())
for item in local_itemlist:
title = item.fulltitle
# Check if the found title matches the release year
year_match = re.search('\(.*(\d{4})\)', title)
if year_match:
found_year = int(year_match.group(1))
title = title[:year_match.start()] + title[year_match.end():]
if title_year > 0 and abs(found_year - title_year) > 1:
continue
# Clean up a bit the returned title to improve the fuzzy matching
title = re.sub(r'\(\d\.\d\)', '', title) # Rating, es: (8.4)
title = re.sub(r'(?i) (film|streaming|ITA)', '', title) # Common keywords in titles
title = re.sub(r'[\[(](HD|B/N)[\])]', '', title) # Common keywords in titles, es. [HD], (B/N), etc.
title = re.sub(r'(?i)\[/?COLOR[^\]]*\]', '', title) # Formatting keywords
# Check if the found title fuzzy matches the searched one
fuzzy = fuzz.token_sort_ratio(mostra, title)
if fuzzy <= 85:
continue
itemlist.append(item)
itemlist = sorted(itemlist, key=lambda item: item.fulltitle)
if show_dialog:
progreso.close()
return itemlist
def do_channels_search(item):
logger.info("streamondemand.channels.database do_channels_search " + item.extra)
try:
title_year = int(item.extra[0:4])
except Exception:
title_year = 0
title_search = item.extra[4:]
import glob
import imp
from lib.fuzzywuzzy import fuzz
master_exclude_data_file = os.path.join( config.get_runtime_path() , "resources", "sodsearch.txt")
logger.info("streamondemand.channels.database master_exclude_data_file=" + master_exclude_data_file)
exclude_data_file = os.path.join( config.get_data_path() , "sodsearch.txt")
logger.info("streamondemand.channels.database exclude_data_file=" + exclude_data_file)
channels_path = os.path.join( config.get_runtime_path() , "channels" , '*.py' )
logger.info("streamondemand.channels.database channels_path=" + channels_path)
channels_excluded = "seriesly\nbuscador\ntengourl\n__init__\n"
for path in [master_exclude_data_file, exclude_data_file]:
if os.path.exists(path):
logger.info("streamondemand.channels.database found exclusion file %s" % path)
fileexclude = open(path, "r")
channels_excluded += fileexclude.read()
fileexclude.close()
else:
logger.info("streamondemand.channels.database not found exclusion file %s" % path)
try:
import xbmcgui
progress_dialog = xbmcgui.DialogProgress()
progress_dialog.create(NLS_Looking_For % item.title)
show_dialog = True
except:
show_dialog = False
channel_files = glob.glob(channels_path)
number_of_channels = len(channel_files)
itemlist = []
channels_successfull = ''
for index, infile in enumerate(channel_files):
if progress_dialog.iscanceled():
logger.info("streamondemand.channels.database channels search aborted")
break
basename = os.path.basename(infile)[:-3]
if basename in channels_excluded:
logger.info("streamondemand.channels.database excluded channel %s" % basename)
else:
if show_dialog:
progress_dialog.update(index * 100 / number_of_channels,
NLS_Searching_In % basename,
NLS_Found_So_Far % (len(itemlist), channels_successfull))
try:
obj = imp.load_source(basename, infile)
logger.info("streamondemand.channels.database loaded %s from %s" % (basename, infile))
#
# Please note that python threads cannot be stopped, therefore, if the channel is
# taking too long, the threading allows the calling process to continue but not
# to kill the channel thread. The runaway channel will continue to execute and
# eventually terminate later. OK, it takes resources, but better to consume some
# additional resources than blocking the KODI GUI altogether.
#
from threading import Thread
class ChannelThread(Thread):
def __init__(self, channel_obj, search_terms):
Thread.__init__(self)
self._channel_obj = channel_obj
self._search_terms = search_terms
self._return = []
def run(self):
self._return = self._channel_obj.search(Item(), self._search_terms)
def join(self, timeout=0):
Thread.join(self, timeout)
if Thread.is_alive(self) and timeout > 0:
logger.info("streamondemand.channels.database forgetting channel %s because is taking more than %s seconds" % (basename, timeout))
return self._return
logger.info("streamondemand.channels.database searching in channel %s for '%s'" % (basename, title_search))
channel_thread = ChannelThread(obj, title_search.replace(' ', '+'))
# NOTE: setting dameon to True allows the main thread can exit even if there are daemon threads still running
channel_thread.daemon = True
channel_thread.start()
channel_result_itemlist = channel_thread.join(60)
for item in channel_result_itemlist:
title = item.fulltitle
# Check if the found title matches the release year
year_match = re.search('\(.*(\d{4})\)', title)
if year_match:
found_year = int(year_match.group(1))
title = title[:year_match.start()] + title[year_match.end():]
if title_year > 0 and abs(found_year - title_year) > 1:
logger.info("streamondemand.channels.database %s: '%s' doesn't match the searched title '%s' %d (delta year is %d)" \
% (basename, item.fulltitle, title_search, title_year, abs(found_year - title_year)))
continue
# Clean up a bit the returned title to improve the fuzzy matching
title = re.sub(r'\(\d\.\d\)', '', title) # Rating, es: (8.4)
title = re.sub(r'(?i) (film|streaming|ITA)', '', title) # Common keywords in titles
title = re.sub(r'[\[(](HD|B/N)[\])]', '', title) # Common keywords in titles, es. [HD], (B/N), etc.
title = re.sub(r'(?i)\[/?COLOR[^\]]*\]', '', title) # Formatting keywords
# Check if the found title fuzzy matches the searched one
fuzzy = fuzz.token_sort_ratio(title_search, title)
if fuzzy <= 85:
logger.info("streamondemand.channels.database %s: '%s' doesn't match the searched title '%s' %d (title fuzzy comparision is %d)" \
% (basename, item.fulltitle, title_search, title_year, fuzzy))
continue
logger.info("streamondemand.channels.database %s: '%s' matches the searched title '%s' %d (title fuzzy comparision is %d)" \
% (basename, item.fulltitle, title_search, title_year, fuzzy))
item.title = "[COLOR orange][%s][/COLOR] %s" % (basename, item.title)
item.fulltitle = title # Use the clean title for sorting
item.viewmode = "list"
itemlist.append(item)
if basename not in channels_successfull:
if len(channels_successfull): channels_successfull += ', '
channels_successfull += basename
except:
import traceback
logger.error(traceback.format_exc())
itemlist = sorted(itemlist, key=lambda Item: fuzz.token_sort_ratio(title_search, item.fulltitle), reverse=True)
if show_dialog:
progress_dialog.close()
return itemlist
def syncCalibreList(col_read=None, col_toread=None, userid=None):
""" Get the lazylibrarian bookid for each read/toread calibre book so we can map our id to theirs,
and sync current/supplied user's read/toread or supplied read/toread columns to calibre database.
Return message giving totals """
myDB = database.DBConnection()
username = ''
readlist = []
toreadlist = []
if not userid:
cookie = cherrypy.request.cookie
if cookie and 'll_uid' in list(cookie.keys()):
userid = cookie['ll_uid'].value
if userid:
res = myDB.match('SELECT UserName,ToRead,HaveRead,CalibreRead,CalibreToRead,Perms from users where UserID=?',
(userid,))
if res:
username = res['UserName']
if not col_read:
col_read = res['CalibreRead']
if not col_toread:
col_toread = res['CalibreToRead']
toreadlist = getList(res['ToRead'])
readlist = getList(res['HaveRead'])
# suppress duplicates (just in case)
toreadlist = list(set(toreadlist))
readlist = list(set(readlist))
else:
return "Error: Unable to get user column settings for %s" % userid
if not userid:
return "Error: Unable to find current userid"
if not col_read and not col_toread:
return "User %s has no calibre columns set" % username
# check user columns exist in calibre and create if not
res = calibredb('custom_columns')
columns = res[0].split('\n')
custom_columns = []
for column in columns:
if column:
custom_columns.append(column.split(' (')[0])
if col_read not in custom_columns:
added = calibredb('add_custom_column', [col_read, col_read, 'bool'])
if "column created" not in added[0]:
return added
if col_toread not in custom_columns:
added = calibredb('add_custom_column', [col_toread, col_toread, 'bool'])
if "column created" not in added[0]:
return added
nomatch = 0
readcol = ''
toreadcol = ''
map_ctol = {}
map_ltoc = {}
if col_read:
readcol = '*' + col_read
if col_toread:
toreadcol = '*' + col_toread
calibre_list = calibreList(col_read, col_toread)
if not isinstance(calibre_list, list):
# got an error message from calibredb
return '"%s"' % calibre_list
for item in calibre_list:
if toreadcol and toreadcol in item or readcol and readcol in item:
authorname, authorid, added = addAuthorNameToDB(item['authors'], refresh=False, addbooks=False)
if authorname:
if authorname != item['authors']:
logger.debug("Changed authorname for [%s] from [%s] to [%s]" %
(item['title'], item['authors'], authorname))
item['authors'] = authorname
bookid, mtype = find_book_in_db(authorname, item['title'], ignored=False, library='eBook')
if bookid and mtype == "Ignored":
logger.warn("Book %s by %s is marked Ignored in database, importing anyway" %
(item['title'], authorname))
if not bookid:
searchterm = "%s <ll> %s" % (item['title'], authorname)
results = search_for(unaccented(searchterm))
if results:
result = results[0]
if result['author_fuzz'] > lazylibrarian.CONFIG['MATCH_RATIO'] \
and result['book_fuzz'] > lazylibrarian.CONFIG['MATCH_RATIO']:
logger.debug("Found (%s%% %s%%) %s: %s" % (result['author_fuzz'], result['book_fuzz'],
result['authorname'], result['bookname']))
bookid = result['bookid']
import_book(bookid)
if bookid:
# NOTE: calibre bookid is always an integer, lazylibrarian bookid is a string
# (goodreads could be used as an int, but googlebooks can't as it's alphanumeric)
# so convert all dict items to strings for ease of matching.
map_ctol[str(item['id'])] = str(bookid)
map_ltoc[str(bookid)] = str(item['id'])
else:
logger.warn('Calibre Book [%s] by [%s] is not in lazylibrarian database' %
(item['title'], authorname))
nomatch += 1
else:
logger.warn('Calibre Author [%s] not matched in lazylibrarian database' % (item['authors']))
nomatch += 1
# Now check current users lazylibrarian read/toread against the calibre library, warn about missing ones
# which might be books calibre doesn't have, or might be minor differences in author or title
for idlist in [("Read", readlist), ("To_Read", toreadlist)]:
booklist = idlist[1]
for bookid in booklist:
cmd = "SELECT AuthorID,BookName from books where BookID=?"
book = myDB.match(cmd, (bookid,))
if not book:
logger.error('Error finding bookid %s' % bookid)
else:
cmd = "SELECT AuthorName from authors where AuthorID=?"
author = myDB.match(cmd, (book['AuthorID'],))
if not author:
logger.error('Error finding authorid %s' % book['AuthorID'])
else:
match = False
high = 0
highname = ''
for item in calibre_list:
if item['authors'] == author['AuthorName'] and item['title'] == book['BookName']:
logger.debug("Exact match for %s [%s]" % (idlist[0], book['BookName']))
map_ctol[str(item['id'])] = str(bookid)
map_ltoc[str(bookid)] = str(item['id'])
match = True
break
if not match:
highid = ''
for item in calibre_list:
if item['authors'] == author['AuthorName']:
n = fuzz.token_sort_ratio(item['title'], book['BookName'])
if n > high:
high = n
highname = item['title']
highid = item['id']
if high > 95:
logger.debug("Found ratio match %s%% [%s] for %s [%s]" %
(high, highname, idlist[0], book['BookName']))
map_ctol[str(highid)] = str(bookid)
map_ltoc[str(bookid)] = str(highid)
match = True
if not match:
logger.warn("No match for %s %s by %s in calibre database, closest match %s%% [%s]" %
(idlist[0], book['BookName'], author['AuthorName'], high, highname))
nomatch += 1
logger.debug("BookID mapping complete, %s match %s, nomatch %s" % (username, len(map_ctol), nomatch))
# now sync the lists
if not userid:
msg = "No userid found"
else:
last_read = []
last_toread = []
calibre_read = []
calibre_toread = []
cmd = 'select SyncList from sync where UserID=? and Label=?'
res = myDB.match(cmd, (userid, col_read))
if res:
last_read = getList(res['SyncList'])
res = myDB.match(cmd, (userid, col_toread))
if res:
last_toread = getList(res['SyncList'])
for item in calibre_list:
if toreadcol and toreadcol in item and item[toreadcol]: # only if True
if str(item['id']) in map_ctol:
calibre_toread.append(map_ctol[str(item['id'])])
else:
logger.warn("Calibre to_read book %s:%s has no lazylibrarian bookid" %
(item['authors'], item['title']))
if readcol and readcol in item and item[readcol]: # only if True
if str(item['id']) in map_ctol:
calibre_read.append(map_ctol[str(item['id'])])
else:
logger.warn("Calibre read book %s:%s has no lazylibrarian bookid" %
(item['authors'], item['title']))
logger.debug("Found %s calibre read, %s calibre toread" % (len(calibre_read), len(calibre_toread)))
logger.debug("Found %s lazylib read, %s lazylib toread" % (len(readlist), len(toreadlist)))
added_to_ll_toread = list(set(toreadlist) - set(last_toread))
removed_from_ll_toread = list(set(last_toread) - set(toreadlist))
added_to_ll_read = list(set(readlist) - set(last_read))
removed_from_ll_read = list(set(last_read) - set(readlist))
logger.debug("lazylibrarian changes to copy to calibre: %s %s %s %s" % (len(added_to_ll_toread),
len(removed_from_ll_toread), len(added_to_ll_read), len(removed_from_ll_read)))
added_to_calibre_toread = list(set(calibre_toread) - set(last_toread))
removed_from_calibre_toread = list(set(last_toread) - set(calibre_toread))
added_to_calibre_read = list(set(calibre_read) - set(last_read))
removed_from_calibre_read = list(set(last_read) - set(calibre_read))
logger.debug("calibre changes to copy to lazylibrarian: %s %s %s %s" % (len(added_to_calibre_toread),
len(removed_from_calibre_toread), len(added_to_calibre_read), len(removed_from_calibre_read)))
calibre_changes = 0
for item in added_to_calibre_read:
if item not in readlist:
readlist.append(item)
logger.debug("Lazylibrarian marked %s as read" % item)
calibre_changes += 1
for item in added_to_calibre_toread:
if item not in toreadlist:
toreadlist.append(item)
logger.debug("Lazylibrarian marked %s as to_read" % item)
calibre_changes += 1
for item in removed_from_calibre_read:
if item in readlist:
readlist.remove(item)
logger.debug("Lazylibrarian removed %s from read" % item)
calibre_changes += 1
for item in removed_from_calibre_toread:
if item in toreadlist:
toreadlist.remove(item)
logger.debug("Lazylibrarian removed %s from to_read" % item)
calibre_changes += 1
if calibre_changes:
myDB.action('UPDATE users SET ToRead=?,HaveRead=? WHERE UserID=?',
(', '.join(toreadlist), ', '.join(readlist), userid))
ll_changes = 0
for item in added_to_ll_toread:
if item in map_ltoc:
res, err, rc = calibredb('set_custom', [col_toread, map_ltoc[item], 'true'], [])
if rc:
msg = "calibredb set_custom error: "
if err:
logger.error(msg + err)
elif res:
logger.error(msg + res)
else:
logger.error(msg + str(rc))
else:
ll_changes += 1
else:
logger.warn("Unable to set calibre %s true for %s" % (col_toread, item))
for item in removed_from_ll_toread:
if item in map_ltoc:
res, err, rc = calibredb('set_custom', [col_toread, map_ltoc[item], ''], [])
if rc:
msg = "calibredb set_custom error: "
if err:
logger.error(msg + err)
elif res:
logger.error(msg + res)
else:
logger.error(msg + str(rc))
else:
ll_changes += 1
else:
logger.warn("Unable to clear calibre %s for %s" % (col_toread, item))
for item in added_to_ll_read:
if item in map_ltoc:
res, err, rc = calibredb('set_custom', [col_read, map_ltoc[item], 'true'], [])
if rc:
msg = "calibredb set_custom error: "
if err:
logger.error(msg + err)
elif res:
logger.error(msg + res)
else:
logger.error(msg + str(rc))
else:
ll_changes += 1
else:
logger.warn("Unable to set calibre %s true for %s" % (col_read, item))
for item in removed_from_ll_read:
if item in map_ltoc:
res, err, rc = calibredb('set_custom', [col_read, map_ltoc[item], ''], [])
if rc:
msg = "calibredb set_custom error: "
if err:
logger.error(msg + err)
elif res:
logger.error(msg + res)
else:
logger.error(msg + str(rc))
else:
ll_changes += 1
else:
logger.warn("Unable to clear calibre %s for %s" % (col_read, item))
# store current sync list as comparison for next sync
controlValueDict = {"UserID": userid, "Label": col_read}
newValueDict = {"Date": str(time.time()), "Synclist": ', '.join(readlist)}
myDB.upsert("sync", newValueDict, controlValueDict)
controlValueDict = {"UserID": userid, "Label": col_toread}
newValueDict = {"Date": str(time.time()), "Synclist": ', '.join(toreadlist)}
myDB.upsert("sync", newValueDict, controlValueDict)
msg = "%s sync updated: %s calibre, %s lazylibrarian" % (username, ll_changes, calibre_changes)
return msg
def do_channels_search(item):
logger.info("streamondemand.channels.database do_channels_search " + item.extra)
try:
title_year = int(item.extra[0:4])
except Exception:
title_year = 0
title_search = item.extra[4:]
import glob
import imp
from lib.fuzzywuzzy import fuzz
master_exclude_data_file = os.path.join(config.get_runtime_path(), "resources", "sodsearch.txt")
logger.info("streamondemand.channels.database master_exclude_data_file=" + master_exclude_data_file)
exclude_data_file = os.path.join(config.get_data_path(), "sodsearch.txt")
logger.info("streamondemand.channels.database exclude_data_file=" + exclude_data_file)
channels_path = os.path.join(config.get_runtime_path(), "channels", "*.py")
logger.info("streamondemand.channels.database channels_path=" + channels_path)
channels_excluded = "seriesly\nbuscador\ntengourl\n__init__\n"
for path in [master_exclude_data_file, exclude_data_file]:
if os.path.exists(path):
logger.info("streamondemand.channels.database found exclusion file %s" % path)
fileexclude = open(path, "r")
channels_excluded += fileexclude.read()
fileexclude.close()
else:
logger.info("streamondemand.channels.database not found exclusion file %s" % path)
try:
import xbmcgui
progress_dialog = xbmcgui.DialogProgress()
progress_dialog.create(NLS_Looking_For % item.title)
show_dialog = True
except:
show_dialog = False
channel_files = glob.glob(channels_path)
number_of_channels = len(channel_files)
itemlist = []
channels_successfull = ""
for index, infile in enumerate(channel_files):
if progress_dialog.iscanceled():
logger.info("streamondemand.channels.database channels search aborted")
break
basename = os.path.basename(infile)[:-3]
if basename in channels_excluded:
logger.info("streamondemand.channels.database excluded channel %s" % basename)
else:
if show_dialog:
progress_dialog.update(
index * 100 / number_of_channels,
NLS_Searching_In % basename,
NLS_Found_So_Far % (len(itemlist), channels_successfull),
)
try:
obj = imp.load_source(basename, infile)
logger.info("streamondemand.channels.database loaded %s from %s" % (basename, infile))
#
# Please note that python threads cannot be stopped, therefore, if the channel is
# taking too long, the threading allows the calling process to continue but not
# to kill the channel thread. The runaway channel will continue to execute and
# eventually terminate later. OK, it takes resources, but better to consume some
# additional resources than blocking the KODI GUI altogether.
#
from threading import Thread
class ChannelThread(Thread):
def __init__(self, channel_obj, search_terms):
Thread.__init__(self)
self._channel_obj = channel_obj
self._search_terms = search_terms
self._return = []
def run(self):
self._return = self._channel_obj.search(Item(), self._search_terms)
def join(self, timeout=0):
Thread.join(self, timeout)
if Thread.is_alive(self) and timeout > 0:
logger.info(
"streamondemand.channels.database forgetting channel %s because is taking more than %s seconds"
% (basename, timeout)
)
return self._return
logger.info(
"streamondemand.channels.database searching in channel %s for '%s'" % (basename, title_search)
)
channel_thread = ChannelThread(obj, title_search.replace(" ", "+"))
# NOTE: setting dameon to True allows the main thread can exit even if there are daemon threads still running
channel_thread.daemon = True
channel_thread.start()
channel_result_itemlist = channel_thread.join(60)
for item in channel_result_itemlist:
title = item.fulltitle
# Check if the found title matches the release year
year_match = re.search("\(.*(\d{4})\)", title)
if year_match:
found_year = int(year_match.group(1))
title = title[: year_match.start()] + title[year_match.end() :]
if title_year > 0 and abs(found_year - title_year) > 1:
logger.info(
"streamondemand.channels.database %s: '%s' doesn't match the searched title '%s' %d (delta year is %d)"
% (basename, item.fulltitle, title_search, title_year, abs(found_year - title_year))
)
continue
# Clean up a bit the returned title to improve the fuzzy matching
title = re.sub(r"\(\d\.\d\)", "", title) # Rating, es: (8.4)
title = re.sub(r"(?i) (film|streaming|ITA)", "", title) # Common keywords in titles
title = re.sub(r"[\[(](HD|B/N)[\])]", "", title) # Common keywords in titles, es. [HD], (B/N), etc.
title = re.sub(r"(?i)\[/?COLOR[^\]]*\]", "", title) # Formatting keywords
# Check if the found title fuzzy matches the searched one
fuzzy = fuzz.token_sort_ratio(title_search, title)
if fuzzy <= 85:
logger.info(
"streamondemand.channels.database %s: '%s' doesn't match the searched title '%s' %d (title fuzzy comparision is %d)"
% (basename, item.fulltitle, title_search, title_year, fuzzy)
)
continue
logger.info(
"streamondemand.channels.database %s: '%s' matches the searched title '%s' %d (title fuzzy comparision is %d)"
% (basename, item.fulltitle, title_search, title_year, fuzzy)
)
item.title = "[COLOR orange][%s][/COLOR] %s" % (basename, item.title)
item.fulltitle = title # Use the clean title for sorting
item.viewmode = "list"
itemlist.append(item)
if basename not in channels_successfull:
if len(channels_successfull):
channels_successfull += ", "
channels_successfull += basename
except:
import traceback
logger.error(traceback.format_exc())
itemlist = sorted(itemlist, key=lambda Item: fuzz.token_sort_ratio(title_search, item.fulltitle), reverse=True)
if show_dialog:
progress_dialog.close()
return itemlist
