def channel_search(queue, channel_parameters, category, title_year, tecleado): try: search_results = [] exec "from channels import " + channel_parameters["channel"] + " as module" mainlist = module.mainlist(Item(channel=channel_parameters["channel"])) for item in mainlist: if item.action != "search" or category and item.extra != category: continue for res_item in module.search(item.clone(), tecleado): title = res_item.fulltitle # If the release year is known, check if it matches the year found in the title if title_year > 0: year_match = re.search('\(.*(\d{4}).*\)', title) if year_match and abs(int(year_match.group(1)) - title_year) > 1: continue # Clean up a bit the returned title to improve the fuzzy matching title = re.sub(r'\(.*\)', '', title) # Anything within () title = re.sub(r'\[.*\]', '', title) # Anything within [] # Check if the found title fuzzy matches the searched one if fuzz.token_sort_ratio(tecleado, title) > 85: res_item.title = "[COLOR azure]" + res_item.title + "[/COLOR][COLOR orange] su [/COLOR][COLOR green]" + channel_parameters["title"] + "[/COLOR]" search_results.append(res_item) queue.put(search_results) except: logger.error("No se puede buscar en: " + channel_parameters["title"]) import traceback logger.error(traceback.format_exc())
def agruparXcontenido(list_result_canal, categoria): global channels_ID_name dict_contenidos = {} list_result = [] for i in list_result_canal: # Formatear titulo i.title = get_title(i) # Eliminar tildes y otros caracteres especiales para la key import unicodedata try: newKey = i.title.lower().strip().decode("UTF-8") newKey = ''.join((c for c in unicodedata.normalize('NFD', newKey) if unicodedata.category(c) != 'Mn')) except: newKey = i.title found = False for k in dict_contenidos.keys(): if fuzz.token_sort_ratio(newKey, k) > 85: # Si el contenido ya estaba en el diccionario añadirlo a la lista de opciones... dict_contenidos[k].append(i) found = True break if not found: # ...sino añadirlo al diccionario dict_contenidos[newKey] = [i] # Añadimos el contenido encontrado en la lista list_result for v in dict_contenidos.values(): title = v[0].title if len(v) > 1: # Eliminar de la lista de nombres de canales los q esten duplicados canales_no_duplicados = [] for i in v: if not i.channel in canales_no_duplicados: canales_no_duplicados.append(channels_ID_name[i.channel]) if len(canales_no_duplicados) > 1: canales = ', '.join([i for i in canales_no_duplicados[:-1]]) title += " (In %s e %s)" % (canales, canales_no_duplicados[-1]) else: title += " (In %s)" % (', '.join( [i for i in canales_no_duplicados])) newItem = v[0].clone(channel="novedades", title=title, action="ver_canales", sub_list=[i.tourl() for i in v], extra=channels_ID_name) else: newItem = v[0].clone(title=title) newItem.text_color = color3 list_result.append(newItem) return sorted(list_result, key=lambda i: i.title.lower())
def do_channels_search(item): logger.info("streamondemand.channels.biblioteca do_channels_search") try: title_year = int(item.extra[0:4]) except: title_year = 0 mostra = item.extra[4:] tecleado = urllib.quote_plus(mostra) itemlist = [] import os import glob import imp from lib.fuzzywuzzy import fuzz import threading import Queue import time master_exclude_data_file = os.path.join(config.get_runtime_path(), "resources", "sodsearch.txt") logger.info("streamondemand.channels.buscador master_exclude_data_file=" + master_exclude_data_file) channels_path = os.path.join(config.get_runtime_path(), "channels", '*.py') logger.info("streamondemand.channels.buscador channels_path=" + channels_path) excluir = "" if os.path.exists(master_exclude_data_file): logger.info("streamondemand.channels.buscador Encontrado fichero exclusiones") fileexclude = open(master_exclude_data_file, "r") excluir = fileexclude.read() fileexclude.close() else: logger.info("streamondemand.channels.buscador No encontrado fichero exclusiones") excluir = "seriesly\nbuscador\ntengourl\n__init__" if config.is_xbmc(): show_dialog = True try: import xbmcgui progreso = xbmcgui.DialogProgressBG() progreso.create(NLS_Looking_For % mostra) except: show_dialog = False def worker(infile, queue): channel_result_itemlist = [] try: basename_without_extension = os.path.basename(infile)[:-3] # http://docs.python.org/library/imp.html?highlight=imp#module-imp obj = imp.load_source(basename_without_extension, infile) logger.info("streamondemand.channels.buscador cargado " + basename_without_extension + " de " + infile) # item.url contains search type: serie, anime, etc... channel_result_itemlist.extend(obj.search(Item(extra=item.url), tecleado)) for local_item in channel_result_itemlist: local_item.title = " [COLOR azure] " + local_item.title + " [/COLOR] [COLOR orange]su[/COLOR] [COLOR green]" + basename_without_extension + "[/COLOR]" local_item.viewmode = "list" except: import traceback logger.error(traceback.format_exc()) queue.put(channel_result_itemlist) channel_files = [infile for infile in glob.glob(channels_path) if os.path.basename(infile)[:-3] not in excluir] result = Queue.Queue() threads = [threading.Thread(target=worker, args=(infile, result)) for infile in channel_files] start_time = int(time.time()) for t in threads: t.daemon = True # NOTE: setting dameon to True allows the main thread to exit even if there are threads still running t.start() number_of_channels = len(channel_files) completed_channels = 0 while completed_channels < number_of_channels: delta_time = int(time.time()) - start_time if len(itemlist) <= 0: timeout = None # No result so far,lets the thread to continue working until a result is returned elif delta_time >= TIMEOUT_TOTAL: break # At least a result matching the searched title has been found, lets stop the search else: timeout = TIMEOUT_TOTAL - delta_time # Still time to gather other results if show_dialog: progreso.update(completed_channels * 100 / number_of_channels) try: result_itemlist = result.get(timeout=timeout) completed_channels += 1 except: # Expired timeout raise an exception break for item in result_itemlist: title = item.fulltitle # If the release year is known, check if it matches the year found in the title if title_year > 0: year_match = re.search('\(.*(\d{4}).*\)', title) if year_match and abs(int(year_match.group(1)) - title_year) > 1: continue # Clean up a bit the returned title to improve the fuzzy matching title = re.sub(r'\(.*\)', '', title) # Anything within () title = re.sub(r'\[.*\]', '', title) # Anything within [] # Check if the found title fuzzy matches the searched one if fuzz.token_sort_ratio(mostra, title) > 85: itemlist.append(item) if show_dialog: progreso.close() itemlist = sorted(itemlist, key=lambda item: item.fulltitle) return itemlist
def do_channels_search(item): logger.info("streamondemand-pureita-master.channels.biblioteca do_channels_search") try: title_year = int(item.extra[0:4]) except: title_year = 0 mostra = item.extra[4:] tecleado = urllib.quote_plus(mostra) itemlist = [] channels_path = os.path.join(config.get_runtime_path(), "channels", "*.xml") logger.info("streamondemand-pureita-master.channels.buscador channels_path=" + channels_path) channel_language = config.get_setting("channel_language") logger.info("streamondemand-pureita-master.channels.buscador channel_language=" + channel_language) if channel_language == "": channel_language = "all" logger.info("streamondemand-pureita-master.channels.buscador channel_language=" + channel_language) if config.is_xbmc(): show_dialog = True try: import xbmcgui progreso = xbmcgui.DialogProgressBG() progreso.create(NLS_Looking_For % mostra) except: show_dialog = False def worker(infile, queue): channel_result_itemlist = [] try: basename_without_extension = os.path.basename(infile)[:-4] # http://docs.python.org/library/imp.html?highlight=imp#module-imp obj = imp.load_source(basename_without_extension, infile[:-4] + ".py") logger.info( "streamondemand-pureita-master.channels.buscador cargado " + basename_without_extension + " de " + infile ) # item.url contains search type: serie, anime, etc... channel_result_itemlist.extend(obj.search(Item(extra=item.url), tecleado)) for local_item in channel_result_itemlist: local_item.title = ( " [COLOR azure] " + local_item.title + " [/COLOR] [COLOR orange]su[/COLOR] [COLOR green]" + basename_without_extension + "[/COLOR]" ) local_item.viewmode = "list" except: import traceback logger.error(traceback.format_exc()) queue.put(channel_result_itemlist) channel_files = glob.glob(channels_path) channel_files_tmp = [] for infile in channel_files: basename_without_extension = os.path.basename(infile)[:-4] channel_parameters = channeltools.get_channel_parameters(basename_without_extension) # No busca si es un canal inactivo if channel_parameters["active"] != "true": continue # No busca si es un canal excluido de la busqueda global if channel_parameters["include_in_global_search"] != "true": continue # No busca si es un canal para adultos, y el modo adulto está desactivado if channel_parameters["adult"] == "true" and config.get_setting("adult_mode") == "false": continue # No busca si el canal es en un idioma filtrado if channel_language != "all" and channel_parameters["language"] != channel_language: continue channel_files_tmp.append(infile) channel_files = channel_files_tmp result = Queue.Queue() threads = [threading.Thread(target=worker, args=(infile, result)) for infile in channel_files] start_time = int(time.time()) for t in threads: t.daemon = ( True ) # NOTE: setting dameon to True allows the main thread to exit even if there are threads still running t.start() number_of_channels = len(channel_files) completed_channels = 0 while completed_channels < number_of_channels: delta_time = int(time.time()) - start_time if len(itemlist) <= 0: timeout = None # No result so far,lets the thread to continue working until a result is returned elif delta_time >= TIMEOUT_TOTAL: break # At least a result matching the searched title has been found, lets stop the search else: timeout = TIMEOUT_TOTAL - delta_time # Still time to gather other results if show_dialog: progreso.update(completed_channels * 100 / number_of_channels) try: result_itemlist = result.get(timeout=timeout) completed_channels += 1 except: # Expired timeout raise an exception break for item in result_itemlist: title = item.fulltitle # If the release year is known, check if it matches the year found in the title if title_year > 0: year_match = re.search("\(.*(\d{4}).*\)", title) if year_match and abs(int(year_match.group(1)) - title_year) > 1: continue # Clean up a bit the returned title to improve the fuzzy matching title = re.sub(r"\(.*\)", "", title) # Anything within () title = re.sub(r"\[.*\]", "", title) # Anything within [] # Check if the found title fuzzy matches the searched one if fuzz.token_sort_ratio(mostra, title) > 85: itemlist.append(item) if show_dialog: progreso.close() itemlist = sorted(itemlist, key=lambda item: item.fulltitle) return itemlist
def do_channels_search(item): logger.info("streamondemand.channels.biblioteca do_channels_search") try: title_year = int(item.extra[0:4]) except: title_year = 0 mostra = item.extra[4:] tecleado = urllib.quote_plus(mostra) itemlist = [] import os import glob import imp from lib.fuzzywuzzy import fuzz import threading import Queue import time master_exclude_data_file = os.path.join(config.get_runtime_path(), "resources", "sodsearch.txt") logger.info("streamondemand.channels.buscador master_exclude_data_file=" + master_exclude_data_file) channels_path = os.path.join(config.get_runtime_path(), "channels", '*.py') logger.info("streamondemand.channels.buscador channels_path=" + channels_path) excluir = "" if os.path.exists(master_exclude_data_file): logger.info( "streamondemand.channels.buscador Encontrado fichero exclusiones") fileexclude = open(master_exclude_data_file, "r") excluir = fileexclude.read() fileexclude.close() else: logger.info( "streamondemand.channels.buscador No encontrado fichero exclusiones" ) excluir = "seriesly\nbuscador\ntengourl\n__init__" if config.is_xbmc(): show_dialog = True try: import xbmcgui progreso = xbmcgui.DialogProgressBG() progreso.create(NLS_Looking_For % mostra) except: show_dialog = False def worker(infile, queue): channel_result_itemlist = [] try: basename_without_extension = os.path.basename(infile)[:-3] # http://docs.python.org/library/imp.html?highlight=imp#module-imp obj = imp.load_source(basename_without_extension, infile) logger.info("streamondemand.channels.buscador cargado " + basename_without_extension + " de " + infile) # item.url contains search type: serie, anime, etc... channel_result_itemlist.extend( obj.search(Item(extra=item.url), tecleado)) for local_item in channel_result_itemlist: local_item.title = " [COLOR azure] " + local_item.title + " [/COLOR] [COLOR orange]su[/COLOR] [COLOR green]" + basename_without_extension + "[/COLOR]" local_item.viewmode = "list" except: import traceback logger.error(traceback.format_exc()) queue.put(channel_result_itemlist) channel_files = [ infile for infile in glob.glob(channels_path) if os.path.basename(infile)[:-3] not in excluir ] result = Queue.Queue() threads = [ threading.Thread(target=worker, args=(infile, result)) for infile in channel_files ] start_time = int(time.time()) for t in threads: t.daemon = True # NOTE: setting dameon to True allows the main thread to exit even if there are threads still running t.start() number_of_channels = len(channel_files) completed_channels = 0 while completed_channels < number_of_channels: delta_time = int(time.time()) - start_time if len(itemlist) <= 0: timeout = None # No result so far,lets the thread to continue working until a result is returned elif delta_time >= TIMEOUT_TOTAL: break # At least a result matching the searched title has been found, lets stop the search else: timeout = TIMEOUT_TOTAL - delta_time # Still time to gather other results if show_dialog: progreso.update(completed_channels * 100 / number_of_channels) try: result_itemlist = result.get(timeout=timeout) completed_channels += 1 except: # Expired timeout raise an exception break for item in result_itemlist: title = item.fulltitle # If the release year is known, check if it matches the year found in the title if title_year > 0: year_match = re.search('\(.*(\d{4}).*\)', title) if year_match and abs(int(year_match.group(1)) - title_year) > 1: continue # Clean up a bit the returned title to improve the fuzzy matching title = re.sub(r'\(.*\)', '', title) # Anything within () title = re.sub(r'\[.*\]', '', title) # Anything within [] # Check if the found title fuzzy matches the searched one if fuzz.token_sort_ratio(mostra, title) > 85: itemlist.append(item) if show_dialog: progreso.close() itemlist = sorted(itemlist, key=lambda item: item.fulltitle) return itemlist
def search_tor_book(books=None, mags=None): if not (lazylibrarian.USE_TOR): return # rename this thread threading.currentThread().name = "SEARCHTORBOOKS" myDB = database.DBConnection() searchlist = [] searchlist1 = [] if books is None: # We are performing a backlog search searchbooks = myDB.select( 'SELECT BookID, AuthorName, Bookname from books WHERE Status="Wanted"' ) # Clear cache if os.path.exists(".ProviderCache"): for f in os.listdir(".ProviderCache"): os.unlink("%s/%s" % (".ProviderCache", f)) # Clearing throttling timeouts t = SimpleCache.ThrottlingProcessor() t.lastRequestTime.clear() else: # The user has added a new book searchbooks = [] if books != False: for book in books: searchbook = myDB.select( 'SELECT BookID, AuthorName, BookName from books WHERE BookID=? AND Status="Wanted"', [book['bookid']]) for terms in searchbook: searchbooks.append(terms) for searchbook in searchbooks: bookid = searchbook[0] author = searchbook[1] book = searchbook[2] dic = { '...': '', '.': ' ', ' & ': ' ', ' = ': ' ', '?': '', '$': 's', ' + ': ' ', '"': '', ',': '', '*': '', ':': '', ';': '' } dicSearchFormatting = {'.': ' +', ' + ': ' '} author = formatter.latinToAscii(formatter.replace_all(author, dic)) book = formatter.latinToAscii(formatter.replace_all(book, dic)) # TRY SEARCH TERM just using author name and book type author = formatter.latinToAscii( formatter.replace_all(author, dicSearchFormatting)) searchterm = author + ' ' + book # + ' ' + lazylibrarian.EBOOK_TYPE searchterm = re.sub('[\.\-\/]', ' ', searchterm).encode('utf-8') searchterm = re.sub(r'\(.*?\)', '', searchterm).encode('utf-8') searchterm = re.sub(r"\s\s+", " ", searchterm) # strip any double white space searchlist.append({ "bookid": bookid, "bookName": searchbook[2], "authorName": searchbook[1], "searchterm": searchterm.strip() }) if not lazylibrarian.KAT: logger.info('No download method is set, use SABnzbd or blackhole') counter = 0 for book in searchlist: #print book.keys() resultlist = providers.IterateOverTorrentSites(book, 'book') #if you can't find teh book specifically, you might find under general search if not resultlist: logger.info( "Searching for type book failed to find any books...moving to general search" ) resultlist = providers.IterateOverTorrentSites(book, 'general') if not resultlist: logger.debug("Adding book %s to queue." % book['searchterm']) else: dictrepl = { '...': '', '.': ' ', ' & ': ' ', ' = ': ' ', '?': '', '$': 's', ' + ': ' ', '"': '', ',': '', '*': '', '(': '', ')': '', '[': '', ']': '', '#': '', '0': '', '1': '', '2': '', '3': '', '4': '', '5': '', '6': '', '7': '', '8': '', '9': '', '\'': '', ':': '', '!': '', '-': '', '\s\s': ' ', ' the ': ' ', ' a ': ' ', ' and ': ' ', ' to ': ' ', ' of ': ' ', ' for ': ' ', ' my ': ' ', ' in ': ' ', ' at ': ' ', ' with ': ' ' } logger.debug(u'searchterm %s' % book['searchterm']) addedCounter = 0 for tor in resultlist: tor_Title = formatter.latinToAscii( formatter.replace_all( str(tor['tor_title']).lower(), dictrepl)).strip() tor_Title = re.sub(r"\s\s+", " ", tor_Title) #remove extra whitespace logger.debug(u'torName %s' % tor_Title) match_ratio = int(lazylibrarian.MATCH_RATIO) tor_Title_match = fuzz.token_sort_ratio( book['searchterm'].lower(), tor_Title) logger.debug("Torrent Title Match %: " + str(tor_Title_match)) if (tor_Title_match > match_ratio): logger.info(u'Found Torrent: %s' % tor['tor_title']) addedCounter = addedCounter + 1 bookid = book['bookid'] tor_Title = (book["authorName"] + ' - ' + book['bookName'] + ' LL.(' + book['bookid'] + ')').strip() tor_url = tor['tor_url'] tor_prov = tor['tor_prov'] tor_size_temp = tor[ 'tor_size'] #Need to cater for when this is NONE (Issue 35) if tor_size_temp is None: tor_size_temp = 1000 tor_size = str(round(float(tor_size_temp) / 1048576, 2)) + ' MB' controlValueDict = {"NZBurl": tor_url} newValueDict = { "NZBprov": tor_prov, "BookID": bookid, "NZBsize": tor_size, "NZBtitle": tor_Title, "Status": "Skipped" } myDB.upsert("wanted", newValueDict, controlValueDict) snatchedbooks = myDB.action( 'SELECT * from books WHERE BookID=? and Status="Snatched"', [bookid]).fetchone() if not snatchedbooks: snatch = DownloadMethod(bookid, tor_prov, tor_Title, tor_url) notifiers.notify_snatch(tor_Title + ' at ' + formatter.now()) break if addedCounter == 0: logger.info("No torrent's found for " + (book["authorName"] + ' ' + book['bookName']).strip() + ". Adding book to queue.") counter = counter + 1 # if not books or books==False: # snatched = searchmag.searchmagazines(mags) # for items in snatched: # snatch = DownloadMethod(items['bookid'], items['tor_prov'], items['tor_title'], items['tor_url']) # notifiers.notify_snatch(items['tor_title']+' at '+formatter.now()) logger.info("Search for Wanted items complete")
def search_tor_book(books=None, mags=None): if not(lazylibrarian.USE_TOR): return # rename this thread threading.currentThread().name = "SEARCHTORBOOKS" myDB = database.DBConnection() searchlist = [] searchlist1 = [] if books is None: # We are performing a backlog search searchbooks = myDB.select('SELECT BookID, AuthorName, Bookname from books WHERE Status="Wanted"') # Clear cache if os.path.exists(".ProviderCache"): for f in os.listdir(".ProviderCache"): os.unlink("%s/%s" % (".ProviderCache", f)) # Clearing throttling timeouts t = SimpleCache.ThrottlingProcessor() t.lastRequestTime.clear() else: # The user has added a new book searchbooks = [] if books != False: for book in books: searchbook = myDB.select('SELECT BookID, AuthorName, BookName from books WHERE BookID=? AND Status="Wanted"', [book['bookid']]) for terms in searchbook: searchbooks.append(terms) for searchbook in searchbooks: bookid = searchbook[0] author = searchbook[1] book = searchbook[2] dic = {'...':'', '.':' ', ' & ':' ', ' = ': ' ', '?':'', '$':'s', ' + ':' ', '"':'', ',':'', '*':'', ':':'', ';':''} dicSearchFormatting = {'.':' +', ' + ':' '} author = formatter.latinToAscii(formatter.replace_all(author, dic)) book = formatter.latinToAscii(formatter.replace_all(book, dic)) # TRY SEARCH TERM just using author name and book type author = formatter.latinToAscii(formatter.replace_all(author, dicSearchFormatting)) searchterm = author + ' ' + book # + ' ' + lazylibrarian.EBOOK_TYPE searchterm = re.sub('[\.\-\/]', ' ', searchterm).encode('utf-8') searchterm = re.sub(r'\(.*?\)', '', searchterm).encode('utf-8') searchterm = re.sub(r"\s\s+" , " ", searchterm) # strip any double white space searchlist.append({"bookid": bookid, "bookName":searchbook[2], "authorName":searchbook[1], "searchterm": searchterm.strip()}) if not lazylibrarian.KAT: logger.info('No download method is set, use SABnzbd or blackhole') counter = 0 for book in searchlist: #print book.keys() resultlist = providers.IterateOverTorrentSites(book,'book') #if you can't find teh book specifically, you might find under general search if not resultlist: logger.info("Searching for type book failed to find any books...moving to general search") resultlist = providers.IterateOverTorrentSites(book,'general') if not resultlist: logger.debug("No result found, Adding book %s to queue " % book['searchterm']) else: dictrepl = {'...':'', '.':' ', ' & ':' ', ' = ': ' ', '?':'', '$':'s', ' + ':' ', '"':'', ',':'', '*':'', '(':'', ')':'', '[':'', ']':'', '#':'', '0':'', '1':'', '2':'', '3':'', '4':'', '5':'', '6':'', '7':'', '8':'' , '9':'', '\'':'', ':':'', '!':'', '-':'', '\s\s':' ', ' the ':' ', ' a ':' ', ' and ':' ', ' to ':' ', ' of ':' ', ' for ':' ', ' my ':' ', ' in ':' ', ' at ':' ', ' with ':' ' } logger.debug(u'searchterm %s' % book['searchterm']) addedCounter = 0 for tor in resultlist: tor_Title = formatter.latinToAscii(formatter.replace_all(str(tor['tor_title']).lower(), dictrepl)).strip() tor_Title = re.sub(r"\s\s+" , " ", tor_Title) #remove extra whitespace logger.debug(u'torName %s' % tor_Title) match_ratio = int(lazylibrarian.MATCH_RATIO) tor_Title_match = fuzz.token_sort_ratio(book['searchterm'].lower(), tor_Title) logger.debug("Torrent Title Match %: " + str(tor_Title_match)) if (tor_Title_match > match_ratio): logger.info(u'Found Torrent: %s' % tor['tor_title']) addedCounter = addedCounter + 1 bookid = book['bookid'] tor_Title = (book["authorName"] + ' - ' + book['bookName'] + ' LL.(' + book['bookid'] + ')').strip() tor_url = tor['tor_url'] tor_prov = tor['tor_prov'] tor_size_temp = tor['tor_size'] #Need to cater for when this is NONE (Issue 35) if tor_size_temp is None: tor_size_temp = 1000 tor_size = str(round(float(tor_size_temp) / 1048576,2))+' MB' controlValueDict = {"NZBurl": tor_url} newValueDict = { "NZBprov": tor_prov, "BookID": bookid, "NZBsize": tor_size, "NZBtitle": tor_Title, "Status": "Skipped" } myDB.upsert("wanted", newValueDict, controlValueDict) snatchedbooks = myDB.action('SELECT * from books WHERE BookID=? and Status="Snatched"', [bookid]).fetchone() if not snatchedbooks: snatch = DownloadMethod(bookid, tor_prov, tor_Title, tor_url) notifiers.notify_snatch(tor_Title+' at '+formatter.now()) break; if addedCounter == 0: logger.info("No torrent's found for " + (book["authorName"] + ' ' + book['bookName']).strip() + ". Adding book to queue.") counter = counter + 1 # if not books or books==False: # snatched = searchmag.searchmagazines(mags) # for items in snatched: # snatch = DownloadMethod(items['bookid'], items['tor_prov'], items['tor_title'], items['tor_url']) # notifiers.notify_snatch(items['tor_title']+' at '+formatter.now()) logger.info("Search for Wanted items complete")
def do_channels_search(item): logger.info("streamondemand-pureita.channels.biblioteca do_channels_search") try: title_year = int(item.extra[0:4]) except: title_year = 0 mostra = item.extra[4:] tecleado = urllib.quote_plus(mostra) itemlist = [] channels_path = os.path.join(config.get_runtime_path(), "channels", '*.xml') logger.info("streamondemand-pureita.channels.buscador channels_path=" + channels_path) channel_language = config.get_setting("channel_language") logger.info("streamondemand-pureita.channels.buscador channel_language=" + channel_language) if channel_language == "": channel_language = "all" logger.info("streamondemand-pureita.channels.buscador channel_language=" + channel_language) if config.is_xbmc(): show_dialog = True try: import xbmcgui progreso = xbmcgui.DialogProgressBG() progreso.create(NLS_Looking_For % mostra) except: show_dialog = False def worker(infile, queue): channel_result_itemlist = [] try: basename_without_extension = os.path.basename(infile)[:-4] # http://docs.python.org/library/imp.html?highlight=imp#module-imp obj = imp.load_source(basename_without_extension, infile[:-4]+".py") logger.info("streamondemand-pureita.channels.buscador cargado " + basename_without_extension + " de " + infile) # item.url contains search type: serie, anime, etc... channel_result_itemlist.extend(obj.search(Item(extra=item.url), tecleado)) for local_item in channel_result_itemlist: local_item.title = " [COLOR azure] " + local_item.title + " [/COLOR] [COLOR orange]su[/COLOR] [COLOR orange]" + basename_without_extension + "[/COLOR]" local_item.viewmode = "list" except: import traceback logger.error(traceback.format_exc()) queue.put(channel_result_itemlist) channel_files = glob.glob(channels_path) channel_files_tmp = [] for infile in channel_files: basename_without_extension = os.path.basename(infile)[:-4] channel_parameters = channeltools.get_channel_parameters(basename_without_extension) # Non cercare se il canale e inattivo if channel_parameters["active"] != "true": continue # Non cercare se un canale e escluso dalla ricerca globale if channel_parameters["include_in_global_search"] != "true": continue # Non cercare se un canale e per adulti e la modalita adulta disabilitata if channel_parameters["adult"] == "true" and config.get_setting("adult_mode") == "false": continue # Non cercare se un canale ha il filtro lingua if channel_language != "all" and channel_parameters["language"] != channel_language: continue channel_files_tmp.append(infile) channel_files = channel_files_tmp result = Queue.Queue() threads = [threading.Thread(target=worker, args=(infile, result)) for infile in channel_files] start_time = int(time.time()) for t in threads: t.daemon = True # NOTE: setting dameon to True allows the main thread to exit even if there are threads still running t.start() number_of_channels = len(channel_files) completed_channels = 0 while completed_channels < number_of_channels: delta_time = int(time.time()) - start_time if len(itemlist) <= 0: timeout = None # No result so far,lets the thread to continue working until a result is returned elif delta_time >= TIMEOUT_TOTAL: break # At least a result matching the searched title has been found, lets stop the search else: timeout = TIMEOUT_TOTAL - delta_time # Still time to gather other results if show_dialog: progreso.update(completed_channels * 100 / number_of_channels) try: result_itemlist = result.get(timeout=timeout) completed_channels += 1 except: # Expired timeout raise an exception break for item in result_itemlist: title = item.fulltitle # If the release year is known, check if it matches the year found in the title if title_year > 0: year_match = re.search('\(.*(\d{4}).*\)', title) if year_match and abs(int(year_match.group(1)) - title_year) > 1: continue # Clean up a bit the returned title to improve the fuzzy matching title = re.sub(r'\(.*\)', '', title) # Anything within () title = re.sub(r'\[.*\]', '', title) # Anything within [] # Check if the found title fuzzy matches the searched one if fuzz.token_sort_ratio(mostra, title) > 85: itemlist.append(item) if show_dialog: progreso.close() itemlist = sorted(itemlist, key=lambda item: item.fulltitle) return itemlist
def syncCalibreList(col_read=None, col_toread=None, userid=None): """ Get the lazylibrarian bookid for each read/toread calibre book so we can map our id to theirs, and sync current/supplied user's read/toread or supplied read/toread columns to calibre database. Return message giving totals """ myDB = database.DBConnection() if not userid: cookie = cherrypy.request.cookie if cookie and 'll_uid' in cookie.keys(): userid = cookie['ll_uid'].value if userid: res = myDB.match( 'SELECT UserName,ToRead,HaveRead,CalibreRead,CalibreToRead,Perms from users where UserID=?', (userid, )) if res: username = res['UserName'] if not col_read: col_read = res['CalibreRead'] if not col_toread: col_toread = res['CalibreToRead'] toreadlist = getList(res['ToRead']) readlist = getList(res['HaveRead']) # suppress duplicates (just in case) toreadlist = list(set(toreadlist)) readlist = list(set(readlist)) else: return "Error: Unable to get user column settings for %s" % userid if not userid: return "Error: Unable to find current userid" if not col_read and not col_toread: return "User %s has no calibre columns set" % username # check user columns exist in calibre and create if not res = calibredb('custom_columns') columns = res[0].split('\n') custom_columns = [] for column in columns: if column: custom_columns.append(column.split(' (')[0]) if col_read not in custom_columns: added = calibredb('add_custom_column', [col_read, col_read, 'bool']) if "column created" not in added[0]: return added if col_toread not in custom_columns: added = calibredb('add_custom_column', [col_toread, col_toread, 'bool']) if "column created" not in added[0]: return added nomatch = 0 readcol = '' toreadcol = '' map_ctol = {} map_ltoc = {} if col_read: readcol = '*' + col_read if col_toread: toreadcol = '*' + col_toread calibre_list = calibreList(col_read, col_toread) if not isinstance(calibre_list, list): # got an error message from calibredb return '"%s"' % calibre_list for item in calibre_list: if toreadcol and toreadcol in item or readcol and readcol in item: authorname, authorid, added = addAuthorNameToDB(item['authors'], refresh=False, addbooks=False) if authorname: if authorname != item['authors']: logger.debug( "Changed authorname for [%s] from [%s] to [%s]" % (item['title'], item['authors'], authorname)) item['authors'] = authorname bookid = find_book_in_db(authorname, item['title']) if not bookid: searchterm = "%s <ll> %s" % (item['title'], authorname) results = search_for(unaccented(searchterm)) if results: result = results[0] if result['author_fuzz'] > lazylibrarian.CONFIG['MATCH_RATIO'] \ and result['book_fuzz'] > lazylibrarian.CONFIG['MATCH_RATIO']: logger.debug( "Found (%s%% %s%%) %s: %s" % (result['author_fuzz'], result['book_fuzz'], result['authorname'], result['bookname'])) bookid = result['bookid'] import_book(bookid) if bookid: # NOTE: calibre bookid is always an integer, lazylibrarian bookid is a string # (goodreads could be used as an int, but googlebooks can't as it's alphanumeric) # so convert all dict items to strings for ease of matching. map_ctol[str(item['id'])] = str(bookid) map_ltoc[str(bookid)] = str(item['id']) else: logger.warn( 'Calibre Book [%s] by [%s] is not in lazylibrarian database' % (item['title'], authorname)) nomatch += 1 else: logger.warn( 'Calibre Author [%s] not matched in lazylibrarian database' % (item['authors'])) nomatch += 1 # Now check current users lazylibrarian read/toread against the calibre library, warn about missing ones # which might be books calibre doesn't have, or might be minor differences in author or title for idlist in [("Read", readlist), ("To_Read", toreadlist)]: booklist = idlist[1] for bookid in booklist: cmd = "SELECT AuthorID,BookName from books where BookID=?" book = myDB.match(cmd, (bookid, )) if not book: logger.error('Error finding bookid %s' % bookid) else: cmd = "SELECT AuthorName from authors where AuthorID=?" author = myDB.match(cmd, (book['AuthorID'], )) if not author: logger.error('Error finding authorid %s' % book['AuthorID']) else: match = False for item in calibre_list: if item['authors'] == author['AuthorName'] and item[ 'title'] == book['BookName']: logger.debug("Exact match for %s [%s]" % (idlist[0], book['BookName'])) map_ctol[str(item['id'])] = str(bookid) map_ltoc[str(bookid)] = str(item['id']) match = True break if not match: high = 0 highname = '' highid = '' for item in calibre_list: if item['authors'] == author['AuthorName']: n = fuzz.token_sort_ratio( item['title'], book['BookName']) if n > high: high = n highname = item['title'] highid = item['id'] if high > 95: logger.debug( "Found ratio match %s%% [%s] for %s [%s]" % (high, highname, idlist[0], book['BookName'])) map_ctol[str(highid)] = str(bookid) map_ltoc[str(bookid)] = str(highid) match = True if not match: logger.warn( "No match for %s %s by %s in calibre database, closest match %s%% [%s]" % (idlist[0], book['BookName'], author['AuthorName'], high, highname)) nomatch += 1 logger.debug("BookID mapping complete, %s match %s, nomatch %s" % (username, len(map_ctol), nomatch)) # now sync the lists if userid: last_read = [] last_toread = [] calibre_read = [] calibre_toread = [] cmd = 'select SyncList from sync where UserID=? and Label=?' res = myDB.match(cmd, (userid, col_read)) if res: last_read = getList(res['SyncList']) res = myDB.match(cmd, (userid, col_toread)) if res: last_toread = getList(res['SyncList']) for item in calibre_list: if toreadcol and toreadcol in item and item[ toreadcol]: # only if True if str(item['id']) in map_ctol: calibre_toread.append(map_ctol[str(item['id'])]) else: logger.warn( "Calibre to_read book %s:%s has no lazylibrarian bookid" % (item['authors'], item['title'])) if readcol and readcol in item and item[readcol]: # only if True if str(item['id']) in map_ctol: calibre_read.append(map_ctol[str(item['id'])]) else: logger.warn( "Calibre read book %s:%s has no lazylibrarian bookid" % (item['authors'], item['title'])) logger.debug("Found %s calibre read, %s calibre toread" % (len(calibre_read), len(calibre_toread))) logger.debug("Found %s lazylib read, %s lazylib toread" % (len(readlist), len(toreadlist))) added_to_ll_toread = list(set(toreadlist) - set(last_toread)) removed_from_ll_toread = list(set(last_toread) - set(toreadlist)) added_to_ll_read = list(set(readlist) - set(last_read)) removed_from_ll_read = list(set(last_read) - set(readlist)) logger.debug("lazylibrarian changes to copy to calibre: %s %s %s %s" % (len(added_to_ll_toread), len(removed_from_ll_toread), len(added_to_ll_read), len(removed_from_ll_read))) added_to_calibre_toread = list(set(calibre_toread) - set(last_toread)) removed_from_calibre_toread = list( set(last_toread) - set(calibre_toread)) added_to_calibre_read = list(set(calibre_read) - set(last_read)) removed_from_calibre_read = list(set(last_read) - set(calibre_read)) logger.debug( "calibre changes to copy to lazylibrarian: %s %s %s %s" % (len(added_to_calibre_toread), len(removed_from_calibre_toread), len(added_to_calibre_read), len(removed_from_calibre_read))) calibre_changes = 0 for item in added_to_calibre_read: if item not in readlist: readlist.append(item) logger.debug("Lazylibrarian marked %s as read" % item) calibre_changes += 1 for item in added_to_calibre_toread: if item not in toreadlist: toreadlist.append(item) logger.debug("Lazylibrarian marked %s as to_read" % item) calibre_changes += 1 for item in removed_from_calibre_read: if item in readlist: readlist.remove(item) logger.debug("Lazylibrarian removed %s from read" % item) calibre_changes += 1 for item in removed_from_calibre_toread: if item in toreadlist: toreadlist.remove(item) logger.debug("Lazylibrarian removed %s from to_read" % item) calibre_changes += 1 if calibre_changes: myDB.action('UPDATE users SET ToRead=?,HaveRead=? WHERE UserID=?', (', '.join(toreadlist), ', '.join(readlist), userid)) ll_changes = 0 for item in added_to_ll_toread: if item in map_ltoc: res, err, rc = calibredb('set_custom', [col_toread, map_ltoc[item], 'true'], []) if rc: msg = "calibredb set_custom error: " if err: logger.error(msg + err) elif res: logger.error(msg + res) else: logger.error(msg + str(rc)) else: ll_changes += 1 else: logger.warn("Unable to set calibre %s true for %s" % (col_toread, item)) for item in removed_from_ll_toread: if item in map_ltoc: res, err, rc = calibredb('set_custom', [col_toread, map_ltoc[item], ''], []) if rc: msg = "calibredb set_custom error: " if err: logger.error(msg + err) elif res: logger.error(msg + res) else: logger.error(msg + str(rc)) else: ll_changes += 1 else: logger.warn("Unable to clear calibre %s for %s" % (col_toread, item)) for item in added_to_ll_read: if item in map_ltoc: res, err, rc = calibredb('set_custom', [col_read, map_ltoc[item], 'true'], []) if rc: msg = "calibredb set_custom error: " if err: logger.error(msg + err) elif res: logger.error(msg + res) else: logger.error(msg + str(rc)) else: ll_changes += 1 else: logger.warn("Unable to set calibre %s true for %s" % (col_read, item)) for item in removed_from_ll_read: if item in map_ltoc: res, err, rc = calibredb('set_custom', [col_read, map_ltoc[item], ''], []) if rc: msg = "calibredb set_custom error: " if err: logger.error(msg + err) elif res: logger.error(msg + res) else: logger.error(msg + str(rc)) else: ll_changes += 1 else: logger.warn("Unable to clear calibre %s for %s" % (col_read, item)) # store current sync list as comparison for next sync controlValueDict = {"UserID": userid, "Label": col_read} newValueDict = { "Date": str(time.time()), "Synclist": ', '.join(readlist) } myDB.upsert("sync", newValueDict, controlValueDict) controlValueDict = {"UserID": userid, "Label": col_toread} newValueDict = { "Date": str(time.time()), "Synclist": ', '.join(toreadlist) } myDB.upsert("sync", newValueDict, controlValueDict) msg = "%s sync updated: %s calibre, %s lazylibrarian" % ( username, ll_changes, calibre_changes) return msg
if not resultlist: logger.debug("Adding book %s to queue." % book['searchterm']) else: dictrepl = {'...': '', '.': ' ', ' & ': ' ', ' = ': ' ', '?': '', '$': 's', ' + ': ' ', '"': '', ',': '', '*': '', '(': '', ')': '', '[': '', ']': '', '#': '', '0': '', '1': '', '2': '', '3': '', '4': '', '5': '', '6': '', '7': '', '8': '', '9': '', '\'': '', ':': '', '!': '', '-': '', '\s\s': ' ', ' the ': ' ', ' a ': ' ', ' and ': ' ', ' to ': ' ', ' of ': ' ', ' for ': ' ', ' my ': ' ', ' in ': ' ', ' at ': ' ', ' with ': ' '} logger.debug(u'searchterm %s' % book['searchterm']) addedCounter = 0 for tor in resultlist: tor_Title = formatter.latinToAscii(formatter.replace_all(str(tor['tor_title']).lower(), dictrepl)).strip() tor_Title = re.sub(r"\s\s+", " ", tor_Title) # remove extra whitespace logger.debug(u'torName %s' % tor_Title) match_ratio = int(lazylibrarian.MATCH_RATIO) tor_Title_match = fuzz.token_sort_ratio(book['searchterm'].lower(), tor_Title) logger.debug("Torrent Title Match %: " + str(tor_Title_match)) if (tor_Title_match > match_ratio): logger.debug(u'Found Torrent: %s' % tor['tor_title']) addedCounter = addedCounter + 1 bookid = book['bookid'] tor_Title = (book["authorName"] + ' - ' + book['bookName'] + ' LL.(' + book['bookid'] + ')').strip() tor_url = tor['tor_url'] tor_prov = tor['tor_prov'] tor_size_temp = tor['tor_size'] # Need to cater for when this is NONE (Issue 35) if tor_size_temp is None: tor_size_temp = 1000 tor_size = str(round(float(tor_size_temp) / 1048576, 2)) + ' MB'
def search_nzb_book(books=None, mags=None): if not (lazylibrarian.USE_NZB): return # rename this thread threading.currentThread().name = "SEARCHNZBBOOKS" myDB = database.DBConnection() searchlist = [] searchlist1 = [] if books is None: # We are performing a backlog search searchbooks = myDB.select('SELECT BookID, AuthorName, Bookname from books WHERE Status="Wanted"') # Clear cache if os.path.exists(".ProviderCache"): for f in os.listdir(".ProviderCache"): os.unlink("%s/%s" % (".ProviderCache", f)) # Clearing throttling timeouts t = SimpleCache.ThrottlingProcessor() t.lastRequestTime.clear() else: # The user has added a new book searchbooks = [] if books != False: for book in books: searchbook = myDB.select( 'SELECT BookID, AuthorName, BookName from books WHERE BookID=? AND Status="Wanted"', [book["bookid"]], ) for terms in searchbook: searchbooks.append(terms) for searchbook in searchbooks: bookid = searchbook[0] author = searchbook[1] book = searchbook[2] dic = { "...": "", ".": " ", " & ": " ", " = ": " ", "?": "", "$": "s", " + ": " ", '"': "", ",": "", "*": "", ":": "", ";": "", } dicSearchFormatting = {".": " +", " + ": " "} author = formatter.latinToAscii(formatter.replace_all(author, dic)) book = formatter.latinToAscii(formatter.replace_all(book, dic)) # TRY SEARCH TERM just using author name and book type author = formatter.latinToAscii(formatter.replace_all(author, dicSearchFormatting)) searchterm = author + " " + book # + ' ' + lazylibrarian.EBOOK_TYPE searchterm = re.sub("[\.\-\/]", " ", searchterm).encode("utf-8") searchterm = re.sub(r"\(.*?\)", "", searchterm).encode("utf-8") searchterm = re.sub(r"\s\s+", " ", searchterm) # strip any double white space searchlist.append( {"bookid": bookid, "bookName": searchbook[2], "authorName": searchbook[1], "searchterm": searchterm.strip()} ) if not lazylibrarian.SAB_HOST and not lazylibrarian.NZB_DOWNLOADER_BLACKHOLE: logger.info("No download method is set, use SABnzbd or blackhole") # TODO - Move the newznab test to providers.py if not lazylibrarian.NEWZNAB and not lazylibrarian.NEWZNAB2 and not lazylibrarian.USENETCRAWLER: logger.info("No providers are set. try use NEWZNAB.") counter = 0 for book in searchlist: # print book.keys() resultlist = providers.IterateOverNewzNabSites(book, "book") # if you can't find teh book specifically, you might find under general search if not resultlist: logger.info("Searching for type book failed to find any books...moving to general search") resultlist = providers.IterateOverNewzNabSites(book, "general") if not resultlist: logger.debug("Adding book %s to queue." % book["searchterm"]) else: dictrepl = { "...": "", ".": " ", " & ": " ", " = ": " ", "?": "", "$": "s", " + ": " ", '"': "", ",": "", "*": "", "(": "", ")": "", "[": "", "]": "", "#": "", "0": "", "1": "", "2": "", "3": "", "4": "", "5": "", "6": "", "7": "", "8": "", "9": "", "'": "", ":": "", "!": "", "-": "", "\s\s": " ", " the ": " ", " a ": " ", " and ": " ", " to ": " ", " of ": " ", " for ": " ", " my ": " ", " in ": " ", " at ": " ", " with ": " ", } logger.debug(u"searchterm %s" % book["searchterm"]) addedCounter = 0 for nzb in resultlist: nzbTitle = formatter.latinToAscii(formatter.replace_all(str(nzb["nzbtitle"]).lower(), dictrepl)).strip() nzbTitle = re.sub(r"\s\s+", " ", nzbTitle) # remove extra whitespace logger.debug(u"nzbName %s" % nzbTitle) match_ratio = int(lazylibrarian.MATCH_RATIO) nzbTitle_match = fuzz.token_sort_ratio(book["searchterm"].lower(), nzbTitle) logger.debug("NZB Title Match %: " + str(nzbTitle_match)) if nzbTitle_match > match_ratio: logger.info(u"Found NZB: %s" % nzb["nzbtitle"]) addedCounter = addedCounter + 1 bookid = book["bookid"] nzbTitle = (book["authorName"] + " - " + book["bookName"] + " LL.(" + book["bookid"] + ")").strip() nzburl = nzb["nzburl"] nzbprov = nzb["nzbprov"] nzbdate_temp = nzb["nzbdate"] nzbsize_temp = nzb["nzbsize"] # Need to cater for when this is NONE (Issue 35) if nzbsize_temp is None: nzbsize_temp = 1000 nzbsize = str(round(float(nzbsize_temp) / 1048576, 2)) + " MB" nzbdate = formatter.nzbdate2format(nzbdate_temp) controlValueDict = {"NZBurl": nzburl} newValueDict = { "NZBprov": nzbprov, "BookID": bookid, "NZBdate": nzbdate, "NZBsize": nzbsize, "NZBtitle": nzbTitle, "Status": "Skipped", } myDB.upsert("wanted", newValueDict, controlValueDict) snatchedbooks = myDB.action( 'SELECT * from books WHERE BookID=? and Status="Snatched"', [bookid] ).fetchone() if not snatchedbooks: snatch = DownloadMethod(bookid, nzbprov, nzbTitle, nzburl) notifiers.notify_snatch(nzbTitle + " at " + formatter.now()) break if addedCounter == 0: logger.info( "No nzb's found for " + (book["authorName"] + " " + book["bookName"]).strip() + ". Adding book to queue." ) counter = counter + 1 if not books or books == False: snatched = searchmag.searchmagazines(mags) for items in snatched: snatch = DownloadMethod(items["bookid"], items["nzbprov"], items["nzbtitle"], items["nzburl"]) notifiers.notify_snatch(items["nzbtitle"] + " at " + formatter.now()) logger.info("Search for Wanted items complete")
def do_channels_search(item): logger.info("streamondemand.channels.biblioteca do_channels_search") try: title_year = int(item.extra[0:4]) except Exception: title_year = 0 mostra = item.extra[4:] tecleado = urllib.quote_plus(mostra) itemlist = [] import os import glob import imp from lib.fuzzywuzzy import fuzz import threading import Queue master_exclude_data_file = os.path.join(config.get_runtime_path(), "resources", "sodsearch.txt") logger.info("streamondemand.channels.buscador master_exclude_data_file=" + master_exclude_data_file) channels_path = os.path.join(config.get_runtime_path(), "channels", '*.py') logger.info("streamondemand.channels.buscador channels_path=" + channels_path) excluir = "" if os.path.exists(master_exclude_data_file): logger.info("streamondemand.channels.buscador Encontrado fichero exclusiones") fileexclude = open(master_exclude_data_file, "r") excluir = fileexclude.read() fileexclude.close() else: logger.info("streamondemand.channels.buscador No encontrado fichero exclusiones") excluir = "seriesly\nbuscador\ntengourl\n__init__" if config.is_xbmc(): show_dialog = True try: import xbmcgui progreso = xbmcgui.DialogProgressBG() progreso.create(NLS_Looking_For % mostra) except: show_dialog = False def worker(infile, queue): channel_result_itemlist = [] try: basename_without_extension = os.path.basename(infile)[:-3] # http://docs.python.org/library/imp.html?highlight=imp#module-imp obj = imp.load_source(basename_without_extension, infile) logger.info("streamondemand.channels.buscador cargado " + basename_without_extension + " de " + infile) # item.url contains search type: serie, anime, etc... channel_result_itemlist.extend(obj.search(Item(extra=item.url), tecleado)) for local_item in channel_result_itemlist: local_item.title = " [COLOR azure] " + local_item.title + " [/COLOR] [COLOR orange]su[/COLOR] [COLOR green]" + basename_without_extension + "[/COLOR]" local_item.viewmode = "list" except: import traceback logger.error(traceback.format_exc()) queue.put(channel_result_itemlist) channel_files = [infile for infile in glob.glob(channels_path) if os.path.basename(infile)[:-3] not in excluir] result = Queue.Queue() threads = [threading.Thread(target=worker, args=(infile, result)) for infile in channel_files] for t in threads: t.start() number_of_channels = len(channel_files) local_itemlist = [] for index, t in enumerate(threads): percentage = index * 100 / number_of_channels if show_dialog: progreso.update(percentage, NLS_Looking_For % mostra) t.join() local_itemlist.extend(result.get()) for item in local_itemlist: title = item.fulltitle # Check if the found title matches the release year year_match = re.search('\(.*(\d{4})\)', title) if year_match: found_year = int(year_match.group(1)) title = title[:year_match.start()] + title[year_match.end():] if title_year > 0 and abs(found_year - title_year) > 1: continue # Clean up a bit the returned title to improve the fuzzy matching title = re.sub(r'\(\d\.\d\)', '', title) # Rating, es: (8.4) title = re.sub(r'(?i) (film|streaming|ITA)', '', title) # Common keywords in titles title = re.sub(r'[\[(](HD|B/N)[\])]', '', title) # Common keywords in titles, es. [HD], (B/N), etc. title = re.sub(r'(?i)\[/?COLOR[^\]]*\]', '', title) # Formatting keywords # Check if the found title fuzzy matches the searched one fuzzy = fuzz.token_sort_ratio(mostra, title) if fuzzy <= 85: continue itemlist.append(item) itemlist = sorted(itemlist, key=lambda item: item.fulltitle) if show_dialog: progreso.close() return itemlist
def do_channels_search(item): logger.info("streamondemand.channels.database do_channels_search " + item.extra) try: title_year = int(item.extra[0:4]) except Exception: title_year = 0 title_search = item.extra[4:] import glob import imp from lib.fuzzywuzzy import fuzz master_exclude_data_file = os.path.join( config.get_runtime_path() , "resources", "sodsearch.txt") logger.info("streamondemand.channels.database master_exclude_data_file=" + master_exclude_data_file) exclude_data_file = os.path.join( config.get_data_path() , "sodsearch.txt") logger.info("streamondemand.channels.database exclude_data_file=" + exclude_data_file) channels_path = os.path.join( config.get_runtime_path() , "channels" , '*.py' ) logger.info("streamondemand.channels.database channels_path=" + channels_path) channels_excluded = "seriesly\nbuscador\ntengourl\n__init__\n" for path in [master_exclude_data_file, exclude_data_file]: if os.path.exists(path): logger.info("streamondemand.channels.database found exclusion file %s" % path) fileexclude = open(path, "r") channels_excluded += fileexclude.read() fileexclude.close() else: logger.info("streamondemand.channels.database not found exclusion file %s" % path) try: import xbmcgui progress_dialog = xbmcgui.DialogProgress() progress_dialog.create(NLS_Looking_For % item.title) show_dialog = True except: show_dialog = False channel_files = glob.glob(channels_path) number_of_channels = len(channel_files) itemlist = [] channels_successfull = '' for index, infile in enumerate(channel_files): if progress_dialog.iscanceled(): logger.info("streamondemand.channels.database channels search aborted") break basename = os.path.basename(infile)[:-3] if basename in channels_excluded: logger.info("streamondemand.channels.database excluded channel %s" % basename) else: if show_dialog: progress_dialog.update(index * 100 / number_of_channels, NLS_Searching_In % basename, NLS_Found_So_Far % (len(itemlist), channels_successfull)) try: obj = imp.load_source(basename, infile) logger.info("streamondemand.channels.database loaded %s from %s" % (basename, infile)) # # Please note that python threads cannot be stopped, therefore, if the channel is # taking too long, the threading allows the calling process to continue but not # to kill the channel thread. The runaway channel will continue to execute and # eventually terminate later. OK, it takes resources, but better to consume some # additional resources than blocking the KODI GUI altogether. # from threading import Thread class ChannelThread(Thread): def __init__(self, channel_obj, search_terms): Thread.__init__(self) self._channel_obj = channel_obj self._search_terms = search_terms self._return = [] def run(self): self._return = self._channel_obj.search(Item(), self._search_terms) def join(self, timeout=0): Thread.join(self, timeout) if Thread.is_alive(self) and timeout > 0: logger.info("streamondemand.channels.database forgetting channel %s because is taking more than %s seconds" % (basename, timeout)) return self._return logger.info("streamondemand.channels.database searching in channel %s for '%s'" % (basename, title_search)) channel_thread = ChannelThread(obj, title_search.replace(' ', '+')) # NOTE: setting dameon to True allows the main thread can exit even if there are daemon threads still running channel_thread.daemon = True channel_thread.start() channel_result_itemlist = channel_thread.join(60) for item in channel_result_itemlist: title = item.fulltitle # Check if the found title matches the release year year_match = re.search('\(.*(\d{4})\)', title) if year_match: found_year = int(year_match.group(1)) title = title[:year_match.start()] + title[year_match.end():] if title_year > 0 and abs(found_year - title_year) > 1: logger.info("streamondemand.channels.database %s: '%s' doesn't match the searched title '%s' %d (delta year is %d)" \ % (basename, item.fulltitle, title_search, title_year, abs(found_year - title_year))) continue # Clean up a bit the returned title to improve the fuzzy matching title = re.sub(r'\(\d\.\d\)', '', title) # Rating, es: (8.4) title = re.sub(r'(?i) (film|streaming|ITA)', '', title) # Common keywords in titles title = re.sub(r'[\[(](HD|B/N)[\])]', '', title) # Common keywords in titles, es. [HD], (B/N), etc. title = re.sub(r'(?i)\[/?COLOR[^\]]*\]', '', title) # Formatting keywords # Check if the found title fuzzy matches the searched one fuzzy = fuzz.token_sort_ratio(title_search, title) if fuzzy <= 85: logger.info("streamondemand.channels.database %s: '%s' doesn't match the searched title '%s' %d (title fuzzy comparision is %d)" \ % (basename, item.fulltitle, title_search, title_year, fuzzy)) continue logger.info("streamondemand.channels.database %s: '%s' matches the searched title '%s' %d (title fuzzy comparision is %d)" \ % (basename, item.fulltitle, title_search, title_year, fuzzy)) item.title = "[COLOR orange][%s][/COLOR] %s" % (basename, item.title) item.fulltitle = title # Use the clean title for sorting item.viewmode = "list" itemlist.append(item) if basename not in channels_successfull: if len(channels_successfull): channels_successfull += ', ' channels_successfull += basename except: import traceback logger.error(traceback.format_exc()) itemlist = sorted(itemlist, key=lambda Item: fuzz.token_sort_ratio(title_search, item.fulltitle), reverse=True) if show_dialog: progress_dialog.close() return itemlist
def syncCalibreList(col_read=None, col_toread=None, userid=None): """ Get the lazylibrarian bookid for each read/toread calibre book so we can map our id to theirs, and sync current/supplied user's read/toread or supplied read/toread columns to calibre database. Return message giving totals """ myDB = database.DBConnection() username = '' readlist = [] toreadlist = [] if not userid: cookie = cherrypy.request.cookie if cookie and 'll_uid' in list(cookie.keys()): userid = cookie['ll_uid'].value if userid: res = myDB.match('SELECT UserName,ToRead,HaveRead,CalibreRead,CalibreToRead,Perms from users where UserID=?', (userid,)) if res: username = res['UserName'] if not col_read: col_read = res['CalibreRead'] if not col_toread: col_toread = res['CalibreToRead'] toreadlist = getList(res['ToRead']) readlist = getList(res['HaveRead']) # suppress duplicates (just in case) toreadlist = list(set(toreadlist)) readlist = list(set(readlist)) else: return "Error: Unable to get user column settings for %s" % userid if not userid: return "Error: Unable to find current userid" if not col_read and not col_toread: return "User %s has no calibre columns set" % username # check user columns exist in calibre and create if not res = calibredb('custom_columns') columns = res[0].split('\n') custom_columns = [] for column in columns: if column: custom_columns.append(column.split(' (')[0]) if col_read not in custom_columns: added = calibredb('add_custom_column', [col_read, col_read, 'bool']) if "column created" not in added[0]: return added if col_toread not in custom_columns: added = calibredb('add_custom_column', [col_toread, col_toread, 'bool']) if "column created" not in added[0]: return added nomatch = 0 readcol = '' toreadcol = '' map_ctol = {} map_ltoc = {} if col_read: readcol = '*' + col_read if col_toread: toreadcol = '*' + col_toread calibre_list = calibreList(col_read, col_toread) if not isinstance(calibre_list, list): # got an error message from calibredb return '"%s"' % calibre_list for item in calibre_list: if toreadcol and toreadcol in item or readcol and readcol in item: authorname, authorid, added = addAuthorNameToDB(item['authors'], refresh=False, addbooks=False) if authorname: if authorname != item['authors']: logger.debug("Changed authorname for [%s] from [%s] to [%s]" % (item['title'], item['authors'], authorname)) item['authors'] = authorname bookid, mtype = find_book_in_db(authorname, item['title'], ignored=False, library='eBook') if bookid and mtype == "Ignored": logger.warn("Book %s by %s is marked Ignored in database, importing anyway" % (item['title'], authorname)) if not bookid: searchterm = "%s <ll> %s" % (item['title'], authorname) results = search_for(unaccented(searchterm)) if results: result = results[0] if result['author_fuzz'] > lazylibrarian.CONFIG['MATCH_RATIO'] \ and result['book_fuzz'] > lazylibrarian.CONFIG['MATCH_RATIO']: logger.debug("Found (%s%% %s%%) %s: %s" % (result['author_fuzz'], result['book_fuzz'], result['authorname'], result['bookname'])) bookid = result['bookid'] import_book(bookid) if bookid: # NOTE: calibre bookid is always an integer, lazylibrarian bookid is a string # (goodreads could be used as an int, but googlebooks can't as it's alphanumeric) # so convert all dict items to strings for ease of matching. map_ctol[str(item['id'])] = str(bookid) map_ltoc[str(bookid)] = str(item['id']) else: logger.warn('Calibre Book [%s] by [%s] is not in lazylibrarian database' % (item['title'], authorname)) nomatch += 1 else: logger.warn('Calibre Author [%s] not matched in lazylibrarian database' % (item['authors'])) nomatch += 1 # Now check current users lazylibrarian read/toread against the calibre library, warn about missing ones # which might be books calibre doesn't have, or might be minor differences in author or title for idlist in [("Read", readlist), ("To_Read", toreadlist)]: booklist = idlist[1] for bookid in booklist: cmd = "SELECT AuthorID,BookName from books where BookID=?" book = myDB.match(cmd, (bookid,)) if not book: logger.error('Error finding bookid %s' % bookid) else: cmd = "SELECT AuthorName from authors where AuthorID=?" author = myDB.match(cmd, (book['AuthorID'],)) if not author: logger.error('Error finding authorid %s' % book['AuthorID']) else: match = False high = 0 highname = '' for item in calibre_list: if item['authors'] == author['AuthorName'] and item['title'] == book['BookName']: logger.debug("Exact match for %s [%s]" % (idlist[0], book['BookName'])) map_ctol[str(item['id'])] = str(bookid) map_ltoc[str(bookid)] = str(item['id']) match = True break if not match: highid = '' for item in calibre_list: if item['authors'] == author['AuthorName']: n = fuzz.token_sort_ratio(item['title'], book['BookName']) if n > high: high = n highname = item['title'] highid = item['id'] if high > 95: logger.debug("Found ratio match %s%% [%s] for %s [%s]" % (high, highname, idlist[0], book['BookName'])) map_ctol[str(highid)] = str(bookid) map_ltoc[str(bookid)] = str(highid) match = True if not match: logger.warn("No match for %s %s by %s in calibre database, closest match %s%% [%s]" % (idlist[0], book['BookName'], author['AuthorName'], high, highname)) nomatch += 1 logger.debug("BookID mapping complete, %s match %s, nomatch %s" % (username, len(map_ctol), nomatch)) # now sync the lists if not userid: msg = "No userid found" else: last_read = [] last_toread = [] calibre_read = [] calibre_toread = [] cmd = 'select SyncList from sync where UserID=? and Label=?' res = myDB.match(cmd, (userid, col_read)) if res: last_read = getList(res['SyncList']) res = myDB.match(cmd, (userid, col_toread)) if res: last_toread = getList(res['SyncList']) for item in calibre_list: if toreadcol and toreadcol in item and item[toreadcol]: # only if True if str(item['id']) in map_ctol: calibre_toread.append(map_ctol[str(item['id'])]) else: logger.warn("Calibre to_read book %s:%s has no lazylibrarian bookid" % (item['authors'], item['title'])) if readcol and readcol in item and item[readcol]: # only if True if str(item['id']) in map_ctol: calibre_read.append(map_ctol[str(item['id'])]) else: logger.warn("Calibre read book %s:%s has no lazylibrarian bookid" % (item['authors'], item['title'])) logger.debug("Found %s calibre read, %s calibre toread" % (len(calibre_read), len(calibre_toread))) logger.debug("Found %s lazylib read, %s lazylib toread" % (len(readlist), len(toreadlist))) added_to_ll_toread = list(set(toreadlist) - set(last_toread)) removed_from_ll_toread = list(set(last_toread) - set(toreadlist)) added_to_ll_read = list(set(readlist) - set(last_read)) removed_from_ll_read = list(set(last_read) - set(readlist)) logger.debug("lazylibrarian changes to copy to calibre: %s %s %s %s" % (len(added_to_ll_toread), len(removed_from_ll_toread), len(added_to_ll_read), len(removed_from_ll_read))) added_to_calibre_toread = list(set(calibre_toread) - set(last_toread)) removed_from_calibre_toread = list(set(last_toread) - set(calibre_toread)) added_to_calibre_read = list(set(calibre_read) - set(last_read)) removed_from_calibre_read = list(set(last_read) - set(calibre_read)) logger.debug("calibre changes to copy to lazylibrarian: %s %s %s %s" % (len(added_to_calibre_toread), len(removed_from_calibre_toread), len(added_to_calibre_read), len(removed_from_calibre_read))) calibre_changes = 0 for item in added_to_calibre_read: if item not in readlist: readlist.append(item) logger.debug("Lazylibrarian marked %s as read" % item) calibre_changes += 1 for item in added_to_calibre_toread: if item not in toreadlist: toreadlist.append(item) logger.debug("Lazylibrarian marked %s as to_read" % item) calibre_changes += 1 for item in removed_from_calibre_read: if item in readlist: readlist.remove(item) logger.debug("Lazylibrarian removed %s from read" % item) calibre_changes += 1 for item in removed_from_calibre_toread: if item in toreadlist: toreadlist.remove(item) logger.debug("Lazylibrarian removed %s from to_read" % item) calibre_changes += 1 if calibre_changes: myDB.action('UPDATE users SET ToRead=?,HaveRead=? WHERE UserID=?', (', '.join(toreadlist), ', '.join(readlist), userid)) ll_changes = 0 for item in added_to_ll_toread: if item in map_ltoc: res, err, rc = calibredb('set_custom', [col_toread, map_ltoc[item], 'true'], []) if rc: msg = "calibredb set_custom error: " if err: logger.error(msg + err) elif res: logger.error(msg + res) else: logger.error(msg + str(rc)) else: ll_changes += 1 else: logger.warn("Unable to set calibre %s true for %s" % (col_toread, item)) for item in removed_from_ll_toread: if item in map_ltoc: res, err, rc = calibredb('set_custom', [col_toread, map_ltoc[item], ''], []) if rc: msg = "calibredb set_custom error: " if err: logger.error(msg + err) elif res: logger.error(msg + res) else: logger.error(msg + str(rc)) else: ll_changes += 1 else: logger.warn("Unable to clear calibre %s for %s" % (col_toread, item)) for item in added_to_ll_read: if item in map_ltoc: res, err, rc = calibredb('set_custom', [col_read, map_ltoc[item], 'true'], []) if rc: msg = "calibredb set_custom error: " if err: logger.error(msg + err) elif res: logger.error(msg + res) else: logger.error(msg + str(rc)) else: ll_changes += 1 else: logger.warn("Unable to set calibre %s true for %s" % (col_read, item)) for item in removed_from_ll_read: if item in map_ltoc: res, err, rc = calibredb('set_custom', [col_read, map_ltoc[item], ''], []) if rc: msg = "calibredb set_custom error: " if err: logger.error(msg + err) elif res: logger.error(msg + res) else: logger.error(msg + str(rc)) else: ll_changes += 1 else: logger.warn("Unable to clear calibre %s for %s" % (col_read, item)) # store current sync list as comparison for next sync controlValueDict = {"UserID": userid, "Label": col_read} newValueDict = {"Date": str(time.time()), "Synclist": ', '.join(readlist)} myDB.upsert("sync", newValueDict, controlValueDict) controlValueDict = {"UserID": userid, "Label": col_toread} newValueDict = {"Date": str(time.time()), "Synclist": ', '.join(toreadlist)} myDB.upsert("sync", newValueDict, controlValueDict) msg = "%s sync updated: %s calibre, %s lazylibrarian" % (username, ll_changes, calibre_changes) return msg
def do_channels_search(item): logger.info("streamondemand.channels.database do_channels_search " + item.extra) try: title_year = int(item.extra[0:4]) except Exception: title_year = 0 title_search = item.extra[4:] import glob import imp from lib.fuzzywuzzy import fuzz master_exclude_data_file = os.path.join(config.get_runtime_path(), "resources", "sodsearch.txt") logger.info("streamondemand.channels.database master_exclude_data_file=" + master_exclude_data_file) exclude_data_file = os.path.join(config.get_data_path(), "sodsearch.txt") logger.info("streamondemand.channels.database exclude_data_file=" + exclude_data_file) channels_path = os.path.join(config.get_runtime_path(), "channels", "*.py") logger.info("streamondemand.channels.database channels_path=" + channels_path) channels_excluded = "seriesly\nbuscador\ntengourl\n__init__\n" for path in [master_exclude_data_file, exclude_data_file]: if os.path.exists(path): logger.info("streamondemand.channels.database found exclusion file %s" % path) fileexclude = open(path, "r") channels_excluded += fileexclude.read() fileexclude.close() else: logger.info("streamondemand.channels.database not found exclusion file %s" % path) try: import xbmcgui progress_dialog = xbmcgui.DialogProgress() progress_dialog.create(NLS_Looking_For % item.title) show_dialog = True except: show_dialog = False channel_files = glob.glob(channels_path) number_of_channels = len(channel_files) itemlist = [] channels_successfull = "" for index, infile in enumerate(channel_files): if progress_dialog.iscanceled(): logger.info("streamondemand.channels.database channels search aborted") break basename = os.path.basename(infile)[:-3] if basename in channels_excluded: logger.info("streamondemand.channels.database excluded channel %s" % basename) else: if show_dialog: progress_dialog.update( index * 100 / number_of_channels, NLS_Searching_In % basename, NLS_Found_So_Far % (len(itemlist), channels_successfull), ) try: obj = imp.load_source(basename, infile) logger.info("streamondemand.channels.database loaded %s from %s" % (basename, infile)) # # Please note that python threads cannot be stopped, therefore, if the channel is # taking too long, the threading allows the calling process to continue but not # to kill the channel thread. The runaway channel will continue to execute and # eventually terminate later. OK, it takes resources, but better to consume some # additional resources than blocking the KODI GUI altogether. # from threading import Thread class ChannelThread(Thread): def __init__(self, channel_obj, search_terms): Thread.__init__(self) self._channel_obj = channel_obj self._search_terms = search_terms self._return = [] def run(self): self._return = self._channel_obj.search(Item(), self._search_terms) def join(self, timeout=0): Thread.join(self, timeout) if Thread.is_alive(self) and timeout > 0: logger.info( "streamondemand.channels.database forgetting channel %s because is taking more than %s seconds" % (basename, timeout) ) return self._return logger.info( "streamondemand.channels.database searching in channel %s for '%s'" % (basename, title_search) ) channel_thread = ChannelThread(obj, title_search.replace(" ", "+")) # NOTE: setting dameon to True allows the main thread can exit even if there are daemon threads still running channel_thread.daemon = True channel_thread.start() channel_result_itemlist = channel_thread.join(60) for item in channel_result_itemlist: title = item.fulltitle # Check if the found title matches the release year year_match = re.search("\(.*(\d{4})\)", title) if year_match: found_year = int(year_match.group(1)) title = title[: year_match.start()] + title[year_match.end() :] if title_year > 0 and abs(found_year - title_year) > 1: logger.info( "streamondemand.channels.database %s: '%s' doesn't match the searched title '%s' %d (delta year is %d)" % (basename, item.fulltitle, title_search, title_year, abs(found_year - title_year)) ) continue # Clean up a bit the returned title to improve the fuzzy matching title = re.sub(r"\(\d\.\d\)", "", title) # Rating, es: (8.4) title = re.sub(r"(?i) (film|streaming|ITA)", "", title) # Common keywords in titles title = re.sub(r"[\[(](HD|B/N)[\])]", "", title) # Common keywords in titles, es. [HD], (B/N), etc. title = re.sub(r"(?i)\[/?COLOR[^\]]*\]", "", title) # Formatting keywords # Check if the found title fuzzy matches the searched one fuzzy = fuzz.token_sort_ratio(title_search, title) if fuzzy <= 85: logger.info( "streamondemand.channels.database %s: '%s' doesn't match the searched title '%s' %d (title fuzzy comparision is %d)" % (basename, item.fulltitle, title_search, title_year, fuzzy) ) continue logger.info( "streamondemand.channels.database %s: '%s' matches the searched title '%s' %d (title fuzzy comparision is %d)" % (basename, item.fulltitle, title_search, title_year, fuzzy) ) item.title = "[COLOR orange][%s][/COLOR] %s" % (basename, item.title) item.fulltitle = title # Use the clean title for sorting item.viewmode = "list" itemlist.append(item) if basename not in channels_successfull: if len(channels_successfull): channels_successfull += ", " channels_successfull += basename except: import traceback logger.error(traceback.format_exc()) itemlist = sorted(itemlist, key=lambda Item: fuzz.token_sort_ratio(title_search, item.fulltitle), reverse=True) if show_dialog: progress_dialog.close() return itemlist