def create_mainlist_item(data): category = data["category_dict"] data = data["data"] new_item = Item(channel=category["channel"], action="list_all", first=0, category= category["category"], thumbnail= category["thumbnail"]) for key in data: new_item.__setattr__(key, data[key]) return new_item
def makeItem(): logger.debug() if sys.argv[2]: sp = sys.argv[2].split('&') url = sp[0] item = Item().fromurl(url) if len(sp) > 1: for e in sp[1:]: key, val = e.split('=') if val.lower() == 'false': val = False elif val.lower() == 'true': val = True item.__setattr__(key, val) # If no item, this is mainlist else: item = Item(channel='channelselector', action='getmainlist', viewmode='movie') return item
def run(item=None): # from core.support import dbg;dbg() logger.debug() if not item: # Extract item from sys.argv if sys.argv[2]: sp = sys.argv[2].split('&') url = sp[0] item = Item().fromurl(url) if len(sp) > 1: for e in sp[1:]: key, val = e.split('=') item.__setattr__(key, val) # If no item, this is mainlist else: item = Item(channel="channelselector", action="getmainlist", viewmode="movie") if not config.get_setting('show_once'): if not config.get_all_settings_addon(): logger.error('corrupted settings.xml!!') settings_xml = os.path.join(config.get_data_path(), "settings.xml") settings_bak = os.path.join(config.get_data_path(), "settings.bak") if filetools.exists(settings_bak): filetools.copy(settings_bak, settings_xml, True) logger.info('restored settings.xml from backup') else: filetools.write(settings_xml, '<settings version="2">\n</settings>' ) # resetted settings else: from platformcode import xbmc_videolibrary xbmc_videolibrary.ask_set_content(silent=False) config.set_setting('show_once', True) logger.info(item.tostring()) try: if not config.get_setting('tmdb_active'): config.set_setting('tmdb_active', True) # If item has no action, stops here if item.action == "": logger.debug("Item without action") return # Action for main menu in channelselector elif item.action == "getmainlist": import channelselector itemlist = channelselector.getmainlist() platformtools.render_items(itemlist, item) # Action for channel types on channelselector: movies, series, etc. elif item.action == "getchanneltypes": import channelselector itemlist = channelselector.getchanneltypes() platformtools.render_items(itemlist, item) # Action for channel listing on channelselector elif item.action == "filterchannels": import channelselector itemlist = channelselector.filterchannels(item.channel_type) platformtools.render_items(itemlist, item) # Special action for playing a video from the library elif item.action == "play_from_library": play_from_library(item) return elif item.action == "keymap": from platformcode import keymaptools if item.open: return keymaptools.open_shortcut_menu() else: return keymaptools.set_key() elif item.channel == "infoplus": from platformcode import infoplus return infoplus.Main(item) elif item.channel == "backup": from platformcode import backup return getattr(backup, item.action)(item) elif item.channel == "elementum_download": from platformcode import elementum_download return getattr(elementum_download, item.action)(item) elif item.channel == "shortcuts": from platformcode import shortcuts return getattr(shortcuts, item.action)(item) elif item.channel == "autorenumber": from platformcode import autorenumber return getattr(autorenumber, item.action)(item) elif item.action == "delete_key": from platformcode import keymaptools return keymaptools.delete_key() elif item.action == "script": from core import tmdb if tmdb.drop_bd(): platformtools.dialog_notification( config.get_localized_string(20000), config.get_localized_string(60011), time=2000, sound=False) elif item.action == "itemInfo": platformtools.dialog_textviewer('Item info', item.parent) elif item.action == "open_browser": import webbrowser if not webbrowser.open(item.url): import xbmc if xbmc.getCondVisibility( 'system.platform.linux') and xbmc.getCondVisibility( 'system.platform.android'): # android xbmc.executebuiltin( 'StartAndroidActivity("", "android.intent.action.VIEW", "", "%s")' % (item.url)) else: try: import urllib.request as urllib except ImportError: import urllib short = urllib.urlopen( 'https://u.nu/api.php?action=shorturl&format=simple&url=' + item.url).read().decode('utf-8') platformtools.dialog_ok( config.get_localized_string(20000), config.get_localized_string(70740) % short) # Action in certain channel specified in "action" and "channel" parameters elif item.action == "check_channels": from platformcode import checkhost checkhost.check_channels() else: # Checks if channel exists if os.path.isfile( os.path.join(config.get_runtime_path(), 'channels', item.channel + ".py")): CHANNELS = 'channels' else: CHANNELS = 'specials' channel_file = os.path.join(config.get_runtime_path(), CHANNELS, item.channel + ".py") logger.debug("channel_file= " + channel_file + ' - ' + CHANNELS + ' - ' + item.channel) channel = None if os.path.exists(channel_file): try: channel = __import__('%s.%s' % (CHANNELS, item.channel), None, None, ['%s.%s' % (CHANNELS, item.channel)]) except ImportError: exec("import " + CHANNELS + "." + item.channel + " as channel") logger.info("Running channel %s | %s" % (channel.__name__, channel.__file__)) # Special play action if item.action == "play": # define la info para trakt try: from core import trakt_tools trakt_tools.set_trakt_info(item) except: pass logger.debug("item.action=%s" % item.action.upper()) # logger.debug("item_toPlay: " + "\n" + item.tostring('\n')) # First checks if channel has a "play" function if hasattr(channel, 'play'): logger.debug("Executing channel 'play' method") itemlist = channel.play(item) b_favourite = item.isFavourite # Play should return a list of playable URLS if len(itemlist) > 0 and isinstance(itemlist[0], Item): item = itemlist[0] if b_favourite: item.isFavourite = True platformtools.play_video(item) # Permitir varias calidades desde play en el Channel elif len(itemlist) > 0 and isinstance(itemlist[0], list): item.video_urls = itemlist platformtools.play_video(item) # If not, shows user an error message else: platformtools.dialog_ok( config.get_localized_string(20000), config.get_localized_string(60339)) # If player don't have a "play" function, not uses the standard play from platformtools else: logger.debug("Executing core 'play' method") platformtools.play_video(item) # Special action for findvideos, where the plugin looks for known urls elif item.action == "findvideos": from core import servertools # First checks if channel has a "findvideos" function if hasattr(channel, 'findvideos'): itemlist = getattr(channel, item.action)(item) # If not, uses the generic findvideos function else: logger.debug("No channel 'findvideos' method, " "executing core method") itemlist = servertools.find_video_items(item) if config.get_setting("max_links", "videolibrary") != 0: itemlist = limit_itemlist(itemlist) from platformcode import subtitletools subtitletools.saveSubtitleName(item) platformtools.render_items(itemlist, item) # Special action for adding a movie to the library elif item.action == "add_pelicula_to_library": from core import videolibrarytools videolibrarytools.add_movie(item) # Special action for adding a serie to the library elif item.action == "add_serie_to_library": from core import videolibrarytools videolibrarytools.add_tvshow(item, channel) # Special action for downloading all episodes from a serie elif item.action == "download_all_episodes": from specials import downloads item.action = item.extra del item.extra downloads.save_download(item) # Special action for searching, first asks for the words then call the "search" function elif item.action == "search": # from core.support import dbg;dbg() if filetools.isfile(temp_search_file) and config.get_setting( 'videolibrary_kodi'): itemlist = [] f = filetools.read(temp_search_file) strList = f.split(',') if strList[0] == '[V]' and strList[1] == item.channel: for it in strList: if it and it not in ['[V]', item.channel]: itemlist.append(Item().fromurl(it)) filetools.write(temp_search_file, f[4:]) return platformtools.render_items(itemlist, item) else: filetools.remove(temp_search_file) logger.debug("item.action=%s" % item.action.upper()) from core import channeltools if config.get_setting('last_search'): last_search = channeltools.get_channel_setting( 'Last_searched', 'search', '') else: last_search = '' search_text = platformtools.dialog_input(last_search) if search_text is not None: channeltools.set_channel_setting('Last_searched', search_text, 'search') itemlist = new_search(item.clone(text=search_text), channel) else: return platformtools.render_items(itemlist, item) # For all other actions else: # import web_pdb; web_pdb.set_trace() logger.debug("Executing channel '%s' method" % item.action) itemlist = getattr(channel, item.action)(item) if config.get_setting('trakt_sync'): from core import trakt_tools token_auth = config.get_setting("token_trakt", "trakt") if not token_auth: trakt_tools.auth_trakt() else: import xbmc if not xbmc.getCondVisibility( 'System.HasAddon(script.trakt)' ) and config.get_setting('install_trakt'): trakt_tools.ask_install_script() itemlist = trakt_tools.trakt_check(itemlist) else: config.set_setting('install_trakt', True) platformtools.render_items(itemlist, item) except WebErrorException as e: import traceback from core import scrapertools logger.error(traceback.format_exc()) platformtools.dialog_ok( config.get_localized_string(59985) % e.channel, config.get_localized_string(60013) % e.url) except Exception as e: import traceback from core import scrapertools logger.error(traceback.format_exc()) patron = 'File "' + os.path.join(config.get_runtime_path(), "channels", "").replace("\\", "\\\\") + r'([^.]+)\.py"' Channel = scrapertools.find_single_match(traceback.format_exc(), patron) if Channel or e.__class__ == logger.ChannelScraperException: if item.url: if platformtools.dialog_yesno( config.get_localized_string(60087) % Channel, config.get_localized_string(60014), nolabel='ok', yeslabel=config.get_localized_string(70739)): run(Item(action="open_browser", url=item.url)) else: platformtools.dialog_ok( config.get_localized_string(60087) % Channel, config.get_localized_string(60014)) else: if platformtools.dialog_yesno(config.get_localized_string(60038), config.get_localized_string(60015)): run(Item(channel="setting", action="report_menu"))
def run(item=None): logger.info() if not item: # Extract item from sys.argv if sys.argv[2]: sp = sys.argv[2].split('&') url = sp[0] item = Item().fromurl(url) if len(sp) > 1: for e in sp[1:]: key, val = e.split('=') item.__setattr__(key, val) # If no item, this is mainlist else: if config.get_setting("start_page"): if not config.get_setting("custom_start"): category = config.get_setting("category").lower() item = Item(channel="news", action="novedades", extra=category, mode = 'silent') else: from channels import side_menu item= Item() item = side_menu.check_user_home(item) item.start = True; else: item = Item(channel="channelselector", action="getmainlist", viewmode="movie") if not config.get_setting('show_once'): from platformcode import xbmc_videolibrary xbmc_videolibrary.ask_set_content(1) config.set_setting('show_once', True) logger.info(item.tostring()) try: if not config.get_setting('tmdb_active'): config.set_setting('tmdb_active', True) # If item has no action, stops here if item.action == "": logger.info("Item sin accion") return # Action for main menu in channelselector elif item.action == "getmainlist": import channelselector itemlist = channelselector.getmainlist() platformtools.render_items(itemlist, item) # Action for channel types on channelselector: movies, series, etc. elif item.action == "getchanneltypes": import channelselector itemlist = channelselector.getchanneltypes() platformtools.render_items(itemlist, item) # Action for channel listing on channelselector elif item.action == "filterchannels": import channelselector itemlist = channelselector.filterchannels(item.channel_type) platformtools.render_items(itemlist, item) # Special action for playing a video from the library elif item.action == "play_from_library": play_from_library(item) return elif item.action == "keymap": from platformcode import keymaptools if item.open: return keymaptools.open_shortcut_menu() else: return keymaptools.set_key() elif item.action == "script": from core import tmdb if tmdb.drop_bd(): platformtools.dialog_notification(config.get_localized_string(20000), config.get_localized_string(60011), time=2000, sound=False) # Action in certain channel specified in "action" and "channel" parameters else: # Entry point for a channel is the "mainlist" action, so here we check parental control if item.action == "mainlist": # Parental control # If it is an adult channel, and user has configured pin, asks for it if channeltools.is_adult(item.channel) and config.get_setting("adult_request_password"): tecleado = platformtools.dialog_input("", config.get_localized_string(60334), True) if tecleado is None or tecleado != config.get_setting("adult_password"): return # # Actualiza el canal individual # if (item.action == "mainlist" and item.channel != "channelselector" and # config.get_setting("check_for_channel_updates") == True): # from core import updater # updater.update_channel(item.channel) # Checks if channel exists channel_file = os.path.join(config.get_runtime_path(), 'channels', item.channel + ".py") logger.info("channel_file=%s" % channel_file) channel = None if os.path.exists(channel_file): try: channel = __import__('channels.%s' % item.channel, None, None, ["channels.%s" % item.channel]) except ImportError: exec("import channels." + item.channel + " as channel") logger.info("Running channel %s | %s" % (channel.__name__, channel.__file__)) # Special play action if item.action == "play": #define la info para trakt try: trakt_tools.set_trakt_info(item) except: pass logger.info("item.action=%s" % item.action.upper()) # logger.debug("item_toPlay: " + "\n" + item.tostring('\n')) # First checks if channel has a "play" function if hasattr(channel, 'play'): logger.info("Executing channel 'play' method") itemlist = channel.play(item) b_favourite = item.isFavourite # Play should return a list of playable URLS if len(itemlist) > 0 and isinstance(itemlist[0], Item): item = itemlist[0] if b_favourite: item.isFavourite = True platformtools.play_video(item) # Permitir varias calidades desde play en el canal elif len(itemlist) > 0 and isinstance(itemlist[0], list): item.video_urls = itemlist platformtools.play_video(item) # If not, shows user an error message else: platformtools.dialog_ok(config.get_localized_string(20000), config.get_localized_string(60339)) # If player don't have a "play" function, not uses the standard play from platformtools else: logger.info("Executing core 'play' method") platformtools.play_video(item) # Special action for findvideos, where the plugin looks for known urls elif item.action == "findvideos": # First checks if channel has a "findvideos" function if hasattr(channel, 'findvideos'): itemlist = getattr(channel, item.action)(item) itemlist = servertools.filter_servers(itemlist) # If not, uses the generic findvideos function else: logger.info("No channel 'findvideos' method, " "executing core method") itemlist = servertools.find_video_items(item) if config.get_setting("max_links", "videolibrary") != 0: itemlist = limit_itemlist(itemlist) from platformcode import subtitletools subtitletools.saveSubtitleName(item) platformtools.render_items(itemlist, item) # Special action for adding a movie to the library elif item.action == "add_pelicula_to_library": videolibrarytools.add_movie(item) # Special action for adding a serie to the library elif item.action == "add_serie_to_library": videolibrarytools.add_tvshow(item, channel) # Special action for downloading all episodes from a serie elif item.action == "download_all_episodes": from channels import downloads item.action = item.extra del item.extra downloads.save_download(item) # Special action for searching, first asks for the words then call the "search" function elif item.action == "search": logger.info("item.action=%s" % item.action.upper()) # last_search = "" # last_search_active = config.get_setting("last_search", "search") # if last_search_active: # try: # current_saved_searches_list = list(config.get_setting("saved_searches_list", "search")) # last_search = current_saved_searches_list[0] # except: # pass last_search = channeltools.get_channel_setting('Last_searched', 'search', '') tecleado = platformtools.dialog_input(last_search) if tecleado is not None: channeltools.set_channel_setting('Last_searched', tecleado, 'search') itemlist = channel.search(item, tecleado) else: return platformtools.render_items(itemlist, item) # For all other actions else: logger.info("Executing channel '%s' method" % item.action) itemlist = getattr(channel, item.action)(item) if config.get_setting('trakt_sync'): token_auth = config.get_setting("token_trakt", "trakt") if not token_auth: trakt_tools.auth_trakt() else: import xbmc if not xbmc.getCondVisibility('System.HasAddon(script.trakt)') and config.get_setting( 'install_trakt'): trakt_tools.ask_install_script() itemlist = trakt_tools.trakt_check(itemlist) else: config.set_setting('install_trakt', True) platformtools.render_items(itemlist, item) except urllib2.URLError as e: import traceback logger.error(traceback.format_exc()) # Grab inner and third party errors if hasattr(e, 'reason'): logger.error("Razon del error, codigo: %s | Razon: %s" % (str(e.reason[0]), str(e.reason[1]))) texto = config.get_localized_string(30050) # "No se puede conectar con el sitio web" platformtools.dialog_ok("alfa", texto) # Grab server response errors elif hasattr(e, 'code'): logger.error("Codigo de error HTTP : %d" % e.code) # "El sitio web no funciona correctamente (error http %d)" platformtools.dialog_ok("alfa", config.get_localized_string(30051) % e.code) except WebErrorException as e: import traceback logger.error(traceback.format_exc()) patron = 'File "' + os.path.join(config.get_runtime_path(), "channels", "").replace("\\", "\\\\") + '([^.]+)\.py"' canal = scrapertools.find_single_match(traceback.format_exc(), patron) platformtools.dialog_ok( config.get_localized_string(59985) + canal, config.get_localized_string(60013) %(e)) except: import traceback logger.error(traceback.format_exc()) patron = 'File "' + os.path.join(config.get_runtime_path(), "channels", "").replace("\\", "\\\\") + '([^.]+)\.py"' canal = scrapertools.find_single_match(traceback.format_exc(), patron) try: import xbmc if config.get_platform(True)['num_version'] < 14: log_name = "xbmc.log" else: log_name = "kodi.log" log_message = config.get_localized_string(50004) + xbmc.translatePath("special://logpath") + log_name except: log_message = "" if canal: platformtools.dialog_ok( config.get_localized_string(60087) %canal, config.get_localized_string(60014), log_message) else: platformtools.dialog_ok( config.get_localized_string(60038), config.get_localized_string(60015), log_message)
def scrapeBlock(item, args, block, patron, headers, action, pagination, debug, typeContentDict, typeActionDict, blacklist, search, pag, function, lang): itemlist = [] log("scrapeBlock qui") matches = scrapertools.find_multiple_matches_groups(block, patron) log('MATCHES =', matches) if debug: regexDbg(item, patron, headers, block) known_keys = [ 'url', 'title', 'title2', 'season', 'episode', 'thumb', 'quality', 'year', 'plot', 'duration', 'genere', 'rating', 'type', 'lang' ] # Legenda known_keys per i groups nei patron # known_keys = ['url', 'title', 'title2', 'season', 'episode', 'thumb', 'quality', # 'year', 'plot', 'duration', 'genere', 'rating', 'type', 'lang'] # url = link relativo o assoluto alla pagina titolo film/serie # title = titolo Film/Serie/Anime/Altro # title2 = titolo dell'episodio Serie/Anime/Altro # season = stagione in formato numerico # episode = numero episodio, in formato numerico. # thumb = linkrealtivo o assoluto alla locandina Film/Serie/Anime/Altro # quality = qualità indicata del video # year = anno in formato numerico (4 cifre) # duration = durata del Film/Serie/Anime/Altro # genere = genere del Film/Serie/Anime/Altro. Es: avventura, commedia # rating = punteggio/voto in formato numerico # type = tipo del video. Es. movie per film o tvshow per le serie. Di solito sono discrimanti usati dal sito # lang = lingua del video. Es: ITA, Sub-ITA, Sub, SUB ITA. # AVVERTENZE: Se il titolo è trovato nella ricerca TMDB/TVDB/Altro allora le locandine e altre info non saranno quelle recuperate nel sito.!!!! stagione = '' # per quei siti che hanno la stagione nel blocco ma non nelle puntate for i, match in enumerate(matches): if pagination and (pag - 1) * pagination > i: continue # pagination if pagination and i >= pag * pagination: break # pagination listGroups = match.keys() match = match.values() if len(listGroups) > len(match): # to fix a bug match = list(match) match.extend([''] * (len(listGroups) - len(match))) scraped = {} for kk in known_keys: val = match[listGroups.index(kk)] if kk in listGroups else '' if val and (kk == "url" or kk == 'thumb') and 'http' not in val: val = scrapertools.find_single_match( item.url, 'https?://[a-z0-9.-]+') + val scraped[kk] = val if scraped['season']: stagione = scraped['season'] episode = scraped['season'] + 'x' + scraped['episode'] elif item.season: episode = item.season + 'x' + scraped['episode'] elif item.contentType == 'tvshow' and (scraped['episode'] == '' and scraped['season'] == '' and stagione == ''): item.news = 'season_completed' episode = '' else: episode = re.sub(r'\s-\s|-|x|–|×|×', 'x', scraped['episode']) if scraped['episode'] else '' second_episode = scrapertools.find_single_match( episode, 'x\d+x(\d+)') if second_episode: episode = re.sub(r'(\d+x\d+)x\d+', r'\1-', episode) + second_episode.zfill(2) #episode = re.sub(r'\s-\s|-|x|–|×', 'x', scraped['episode']) if scraped['episode'] else '' title = cleantitle(scraped['title']) if scraped['title'] else '' title2 = cleantitle(scraped['title2']) if scraped['title2'] else '' quality = scraped['quality'].strip() if scraped['quality'] else '' Type = scraped['type'] if scraped['type'] else '' plot = cleantitle(scraped["plot"]) if scraped["plot"] else '' # make formatted Title [longtitle] s = ' - ' title = episode + (s if episode and title else '') + title longtitle = title + (s if title and title2 else '') + title2 longtitle = typo(longtitle, 'bold') longtitle += typo(quality, '_ [] color kod') if quality else '' lang1, longtitle = scrapeLang(scraped, lang, longtitle) # if title is set, probably this is a list of episodes or video sources # necessaria l'aggiunta di == scraped["title"] altrimenti non prende i gruppi dopo le categorie if item.infoLabels["title"] == scraped["title"]: infolabels = item.infoLabels else: infolabels = {} if scraped['year']: infolabels['year'] = scraped['year'] if scraped["plot"]: infolabels['plot'] = plot if scraped['duration']: matches = scrapertools.find_multiple_matches( scraped['duration'], r'([0-9])\s*?(?:[hH]|:|\.|,|\\|\/|\||\s)\s*?([0-9]+)') for h, m in matches: scraped['duration'] = int(h) * 60 + int(m) if not matches: scraped['duration'] = scrapertools.find_single_match( scraped['duration'], r'(\d+)') infolabels['duration'] = int(scraped['duration']) * 60 if scraped['genere']: genres = scrapertools.find_multiple_matches( scraped['genere'], '[A-Za-z]+') infolabels['genere'] = ", ".join(genres) if scraped["rating"]: infolabels['rating'] = scrapertools.decodeHtmlentities( scraped["rating"]) AC = CT = '' if typeContentDict: for name, variants in typeContentDict.items(): if str(scraped['type']).lower() in variants: CT = name break else: CT = item.contentType if typeActionDict: for name, variants in typeActionDict.items(): if str(scraped['type']).lower() in variants: AC = name break else: AC = action if (scraped["title"] not in blacklist) and (search.lower() in longtitle.lower()): it = Item( channel=item.channel, action=AC if AC else action, contentType='episode' if function == 'episodios' else CT if CT else item.contentType, title=longtitle, fulltitle=item.fulltitle if function == 'episodios' else title, show=item.show if function == 'episodios' else title, quality=quality, url=scraped["url"], infoLabels=infolabels, thumbnail=item.thumbnail if function == 'episodios' else scraped["thumb"], args=item.args, contentSerieName=scraped['title'] if item.contentType or CT != 'movie' and function != 'episodios' else item.fulltitle if function == 'episodios' else '', contentTitle=scraped['title'] if item.contentType or CT == 'movie' else '', contentLanguage=lang1, contentEpisodeNumber=episode if episode else '', news=item.news if item.news else '') for lg in list(set(listGroups).difference(known_keys)): it.__setattr__(lg, match[listGroups.index(lg)]) if 'itemHook' in args: it = args['itemHook'](it) itemlist.append(it) return itemlist, matches
def scrapeBlock(item, args, block, patron, headers, action, pagination, debug, typeContentDict, typeActionDict, blacklist, search, pag, function, lang): itemlist = [] log("scrapeBlock qui", block, patron) matches = scrapertoolsV2.find_multiple_matches_groups(block, patron) log('MATCHES =', matches) if debug: regexDbg(item, patron, headers, block) known_keys = [ 'url', 'title', 'title2', 'episode', 'thumb', 'quality', 'year', 'plot', 'duration', 'genere', 'rating', 'type', 'lang' ] # lang = '' # aggiunto per gestire i siti con pagine di serietv dove si hanno i video in ita e in subita for i, match in enumerate(matches): if pagination and (pag - 1) * pagination > i: continue # pagination if pagination and i >= pag * pagination: break # pagination listGroups = match.keys() match = match.values() if len(listGroups) > len(match): # to fix a bug match = list(match) match.extend([''] * (len(listGroups) - len(match))) scraped = {} for kk in known_keys: val = match[listGroups.index(kk)] if kk in listGroups else '' if val and (kk == "url" or kk == 'thumb') and 'http' not in val: val = scrapertoolsV2.find_single_match( item.url, 'https?://[a-z0-9.-]+') + val scraped[kk] = val episode = re.sub(r'\s-\s|-|x|–|×', 'x', scraped['episode']) if scraped['episode'] else '' title = cleantitle(scraped['title']) if scraped['title'] else '' title2 = cleantitle(scraped['title2']) if scraped['title2'] else '' quality = scraped['quality'].strip() if scraped['quality'] else '' Type = scraped['type'] if scraped['type'] else '' plot = cleantitle(scraped["plot"]) if scraped["plot"] else '' # make formatted Title [longtitle] s = ' - ' title = episode + (s if episode and title else '') + title longtitle = title + (s if title and title2 else '') + title2 longtitle = typo(longtitle, 'bold') longtitle += (typo(Type, '_ () bold') if Type else '') + (typo( quality, '_ [] color kod') if quality else '') # # per togliere la voce [ITA] da liste che non siano titoli (es.: genere) # if action != 'peliculas': # lang, longtitle = scrapeLang(scraped, lang, longtitle) # else: # longtitle = longtitle.replace('[ITA]','') # lang = '' lang, longtitle = scrapeLang(scraped, lang, longtitle) # if title is set, probably this is a list of episodes or video sources # necessaria l'aggiunta di == scraped["title"] altrimenti non prende i gruppi dopo le categorie if item.infoLabels["title"] == scraped["title"]: infolabels = item.infoLabels else: infolabels = {} if scraped['year']: infolabels['year'] = scraped['year'] if scraped["plot"]: infolabels['plot'] = plot if scraped['duration']: matches = scrapertoolsV2.find_multiple_matches( scraped['duration'], r'([0-9])\s*?(?:[hH]|:|\.|,|\\|\/|\||\s)\s*?([0-9]+)') for h, m in matches: scraped['duration'] = int(h) * 60 + int(m) if not matches: scraped['duration'] = scrapertoolsV2.find_single_match( scraped['duration'], r'(\d+)') infolabels['duration'] = int(scraped['duration']) * 60 if scraped['genere']: genres = scrapertoolsV2.find_multiple_matches( scraped['genere'], '[A-Za-z]+') infolabels['genere'] = ", ".join(genres) if scraped["rating"]: infolabels['rating'] = scrapertoolsV2.decodeHtmlentities( scraped["rating"]) AC = CT = '' if typeContentDict: for name, variants in typeContentDict.items(): if str(scraped['type']).lower() in variants: CT = name else: CT = item.contentType if typeActionDict: for name, variants in typeActionDict.items(): if str(scraped['type']).lower() in variants: AC = name else: AC = action if (scraped["title"] not in blacklist) and (search.lower() in longtitle.lower()): it = Item( channel=item.channel, action=AC if AC else action, contentType='episode' if function == 'episodios' else CT if CT else item.contentType, title=longtitle, fulltitle=item.fulltitle if function == 'episodios' else title, show=item.show if function == 'episodios' else title, quality=quality, url=scraped["url"], infoLabels=infolabels, thumbnail=item.thumbnail if function == 'episodios' else scraped["thumb"], args=item.args, contentSerieName=title if item.contentType != 'movie' and function != 'episodios' else item.fulltitle if function == 'episodios' else '', contentTitle=title if item.contentType == 'movie' else '', contentLanguage=lang, ep=episode if episode else '') for lg in list(set(listGroups).difference(known_keys)): it.__setattr__(lg, match[listGroups.index(lg)]) if 'itemHook' in args: it = args['itemHook'](it) itemlist.append(it) return itemlist, matches
def scrape(item, patron='', listGroups=[], headers="", blacklist="", data="", patron_block="", patronNext="", action="findvideos", addVideolibrary=True, type_content_dict={}, type_action_dict={}): # patron: the patron to use for scraping page, all capturing group must match with listGroups # listGroups: a list containing the scraping info obtained by your patron, in order # accepted values are: url, title, thumb, quality, year, plot, duration, genre, rating, episode, lang # header: values to pass to request header # blacklist: titles that you want to exclude(service articles for example) # data: if you want to pass data manually, maybe because you need some custom replacement # patron_block: patron to get parts of the page (to scrape with patron attribute), # if you need a "block inside another block" you can create a list, please note that all matches # will be packed as string # patronNext: patron for scraping next page link # action: if you want results perform an action different from "findvideos", useful when scraping film by genres # url_host: string to prepend to scrapedurl, useful when url don't contain host # example usage: # import support # itemlist = [] # patron = 'blablabla' # headers = [['Referer', host]] # blacklist = 'Request a TV serie!' # return support.scrape(item, itemlist, patron, ['thumb', 'quality', 'url', 'title', 'title2', 'year', 'plot', 'episode', 'lang'], # headers=headers, blacklist=blacklist) # listGroups # thumb = immagine, quality = qualità, url = link singolo o gruppo, title = titolo film o serie, title2 = titolo aggiuntivo # year = anno del film o della serie, plot = descrizione film o serie, episode = numero stagione - numero episodio in caso di serie, # lang = lingua del video # 'type' is a check for typologies of content e.g. Film or TV Series # 'episode' is a key to grab episode numbers if it is separated from the title # IMPORTANT 'type' is a special key, to work need type_content_dict={} and type_action_dict={} itemlist = [] if not data: data = httptools.downloadpage(item.url, headers=headers, ignore_response_code=True).data.replace( "'", '"') data = re.sub('\n|\t', ' ', data) # replace all ' with " and eliminate newline, so we don't need to worry about log('DATA =', data) block = data if patron_block: if type(patron_block) == str: patron_block = [patron_block] for n, regex in enumerate(patron_block): blocks = scrapertoolsV2.find_multiple_matches(block, regex) block = "" for b in blocks: block += "\n" + str(b) log('BLOCK ', n, '=', block) else: block = data if patron and listGroups: matches = scrapertoolsV2.find_multiple_matches(block, patron) log('MATCHES =', matches) known_keys = [ 'url', 'title', 'title2', 'episode', 'thumb', 'quality', 'year', 'plot', 'duration', 'genere', 'rating', 'type', 'lang' ] #by greko aggiunto episode lang = '' # aggiunto per gestire i siti con pagine di serietv dove si hanno i video in ita e in subita for match in matches: if len(listGroups) > len(match): # to fix a bug match = list(match) match.extend([''] * (len(listGroups) - len(match))) scraped = {} for kk in known_keys: val = match[listGroups.index(kk)] if kk in listGroups else '' if val and (kk == "url" or kk == 'thumb') and 'http' not in val: val = scrapertoolsV2.find_single_match( item.url, 'https?://[a-z0-9.-]+') + val scraped[kk] = val title = scrapertoolsV2.htmlclean( scrapertoolsV2.decodeHtmlentities(scraped["title"])).replace( '’', '\'').replace('"', "'").strip() # fix by greko da " a ' plot = scrapertoolsV2.htmlclean( scrapertoolsV2.decodeHtmlentities(scraped["plot"])) longtitle = typo(title, 'bold') if scraped['quality']: longtitle = longtitle + typo(scraped['quality'], '_ [] color kod') if scraped['episode']: scraped['episode'] = re.sub(r'\s-\s|-|x|–', 'x', scraped['episode']) longtitle = typo(scraped['episode'] + ' - ', 'bold') + longtitle if scraped['title2']: title2 = scrapertoolsV2.htmlclean( scrapertoolsV2.decodeHtmlentities( scraped["title2"])).replace('"', "'").strip() longtitle = longtitle + typo(title2, 'bold _ -- _') ## Aggiunto/modificato per gestire i siti che hanno i video ## in ita e subita delle serie tv nella stessa pagina if scraped['lang']: if 'sub' in scraped['lang'].lower(): lang = 'Sub-ITA' else: lang = 'ITA' if lang != '': longtitle += typo(lang, '_ [] color kod') if item.infoLabels[ "title"] or item.fulltitle: # if title is set, probably this is a list of episodes or video sources infolabels = item.infoLabels else: infolabels = {} if scraped["year"]: infolabels['year'] = scraped["year"] if scraped["plot"]: infolabels['plot'] = plot if scraped["duration"]: matches = scrapertoolsV2.find_multiple_matches( scraped["duration"], r'([0-9])\s*?(?:[hH]|:|\.|,|\\|\/|\||\s)\s*?([0-9]+)') for h, m in matches: scraped["duration"] = int(h) * 60 + int(m) if not matches: scraped["duration"] = scrapertoolsV2.find_single_match( scraped["duration"], r'(\d+)') infolabels['duration'] = int(scraped["duration"]) * 60 if scraped["genere"]: genres = scrapertoolsV2.find_multiple_matches( scraped["genere"], '[A-Za-z]+') infolabels['genere'] = ", ".join(genres) if scraped["rating"]: infolabels['rating'] = scrapertoolsV2.decodeHtmlentities( scraped["rating"]) if type_content_dict: for name, variants in type_content_dict.items(): if scraped['type'] in variants: item.contentType = name if type_action_dict: for name, variants in type_action_dict.items(): if scraped['type'] in variants: action = name if inspect.stack()[1][3] == 'episodios': item.contentType = 'episode' if scraped["title"] not in blacklist: it = Item(channel=item.channel, action=action, contentType=item.contentType, title=longtitle, fulltitle=title, show=title, language=lang if lang != '' else '', quality=scraped["quality"], url=scraped["url"], infoLabels=infolabels, thumbnail=scraped["thumb"], args=item.args) for lg in list(set(listGroups).difference(known_keys)): it.__setattr__(lg, match[listGroups.index(lg)]) itemlist.append(it) checkHost(item, itemlist) if (item.contentType == "tvshow" and (action != "findvideos" and action != "play")) \ or (item.contentType == "episode" and action != "play") \ or (item.contentType == "movie" and action != "play"): tmdb.set_infoLabels_itemlist(itemlist, seekTmdb=True) else: for it in itemlist: it.infoLabels = item.infoLabels if patronNext: nextPage(itemlist, item, data, patronNext, 2) if addVideolibrary and (item.infoLabels["title"] or item.fulltitle): item.fulltitle = item.infoLabels["title"] videolibrary(itemlist, item) return itemlist
def run(item=None): logger.info() if not item: # Extract item from sys.argv if sys.argv[2]: sp = sys.argv[2].split('&') url = sp[0] item = Item().fromurl(url) if len(sp) > 1: for e in sp[1:]: key, val = e.split('=') item.__setattr__(key, val) # If no item, this is mainlist else: if config.get_setting("start_page"): if not config.get_setting("custom_start"): dictCategory = { config.get_localized_string(70137): 'peliculas', config.get_localized_string(30123): 'series', config.get_localized_string(30124): 'anime', config.get_localized_string(70018): 'infantiles', config.get_localized_string(60513): 'documentales', config.get_localized_string(70013): 'terror', config.get_localized_string(70014): 'castellano', config.get_localized_string(59976): 'latino', config.get_localized_string(70171): 'torrent', } if not config.get_setting("category") in dictCategory.keys(): config.set_setting('category', config.get_localized_string(70137)) category = dictCategory[config.get_setting("category")] item = Item(channel="news", action="novedades", extra=category, mode = 'silent') else: from specials import side_menu item= Item() item = side_menu.check_user_home(item) item.start = True else: item = Item(channel="channelselector", action="getmainlist", viewmode="movie") if not config.get_setting('show_once'): from platformcode import xbmc_videolibrary xbmc_videolibrary.ask_set_content(silent=False) config.set_setting('show_once', True) logger.info(item.tostring()) try: if not config.get_setting('tmdb_active'): config.set_setting('tmdb_active', True) # If item has no action, stops here if item.action == "": logger.info("Item without action") return # Action for main menu in channelselector elif item.action == "getmainlist": import channelselector itemlist = channelselector.getmainlist() platformtools.render_items(itemlist, item) # Action for channel types on channelselector: movies, series, etc. elif item.action == "getchanneltypes": import channelselector itemlist = channelselector.getchanneltypes() platformtools.render_items(itemlist, item) # Action for channel listing on channelselector elif item.action == "filterchannels": import channelselector itemlist = channelselector.filterchannels(item.channel_type) platformtools.render_items(itemlist, item) # Special action for playing a video from the library elif item.action == "play_from_library": play_from_library(item) return elif item.action == "keymap": from platformcode import keymaptools if item.open: return keymaptools.open_shortcut_menu() else: return keymaptools.set_key() elif item.action == "delete_key": from platformcode import keymaptools return keymaptools.delete_key() elif item.action == "script": from core import tmdb if tmdb.drop_bd(): platformtools.dialog_notification(config.get_localized_string(20000), config.get_localized_string(60011), time=2000, sound=False) elif item.action == "itemInfo": import base64 platformtools.dialog_textviewer('Item info', item.parent) elif item.action == "open_browser": import webbrowser if not webbrowser.open(item.url): import xbmc if xbmc.getCondVisibility('system.platform.linux') and xbmc.getCondVisibility('system.platform.android'): # android xbmc.executebuiltin('StartAndroidActivity("", "android.intent.action.VIEW", "", "%s")' % (item.url)) else: try: import urllib.request as urllib except ImportError: import urllib short = urllib.urlopen( 'https://u.nu/api.php?action=shorturl&format=simple&url=' + item.url).read().decode('utf-8') platformtools.dialog_ok(config.get_localized_string(20000), config.get_localized_string(70740) % short) # Action in certain channel specified in "action" and "channel" parameters else: # # Actualiza el canal individual # if (item.action == "mainlist" and item.channel != "channelselector" and # config.get_setting("check_for_channel_updates") == True): # from core import updater # updater.update_channel(item.channel) # Checks if channel exists if os.path.isfile(os.path.join(config.get_runtime_path(), 'channels', item.channel + ".py")): CHANNELS = 'channels' else: CHANNELS = 'specials' channel_file = os.path.join(config.get_runtime_path(), CHANNELS, item.channel + ".py") logger.info("channel_file= " + channel_file + ' - ' + CHANNELS + ' - ' + item.channel) channel = None if os.path.exists(channel_file): try: channel = __import__('%s.%s' % (CHANNELS, item.channel), None, None, ['%s.%s' % (CHANNELS, item.channel)]) except ImportError: exec("import " + CHANNELS + "." + item.channel + " as channel") logger.info("Running channel %s | %s" % (channel.__name__, channel.__file__)) # Special play action if item.action == "play": #define la info para trakt try: from core import trakt_tools trakt_tools.set_trakt_info(item) except: pass logger.info("item.action=%s" % item.action.upper()) # logger.debug("item_toPlay: " + "\n" + item.tostring('\n')) # First checks if channel has a "play" function if hasattr(channel, 'play'): logger.info("Executing channel 'play' method") itemlist = channel.play(item) b_favourite = item.isFavourite # Play should return a list of playable URLS if len(itemlist) > 0 and isinstance(itemlist[0], Item): item = itemlist[0] if b_favourite: item.isFavourite = True platformtools.play_video(item) # Permitir varias calidades desde play en el canal elif len(itemlist) > 0 and isinstance(itemlist[0], list): item.video_urls = itemlist platformtools.play_video(item) # If not, shows user an error message else: platformtools.dialog_ok(config.get_localized_string(20000), config.get_localized_string(60339)) # If player don't have a "play" function, not uses the standard play from platformtools else: logger.info("Executing core 'play' method") platformtools.play_video(item) # Special action for findvideos, where the plugin looks for known urls elif item.action == "findvideos": from core import servertools # First checks if channel has a "findvideos" function if hasattr(channel, 'findvideos'): itemlist = getattr(channel, item.action)(item) itemlist = servertools.filter_servers(itemlist) # If not, uses the generic findvideos function else: logger.info("No channel 'findvideos' method, " "executing core method") itemlist = servertools.find_video_items(item) if config.get_setting("max_links", "videolibrary") != 0: itemlist = limit_itemlist(itemlist) from platformcode import subtitletools subtitletools.saveSubtitleName(item) platformtools.render_items(itemlist, item) # Special action for adding a movie to the library elif item.action == "add_pelicula_to_library": from core import videolibrarytools videolibrarytools.add_movie(item) # Special action for adding a serie to the library elif item.action == "add_serie_to_library": from core import videolibrarytools videolibrarytools.add_tvshow(item, channel) # Special action for downloading all episodes from a serie elif item.action == "download_all_episodes": from specials import downloads item.action = item.extra del item.extra downloads.save_download(item) # Special action for searching, first asks for the words then call the "search" function elif item.action == "search": logger.info("item.action=%s" % item.action.upper()) from core import channeltools if config.get_setting('last_search'): last_search = channeltools.get_channel_setting('Last_searched', 'search', '') else: last_search = '' tecleado = platformtools.dialog_input(last_search) if tecleado is not None: channeltools.set_channel_setting('Last_searched', tecleado, 'search') if 'search' in dir(channel): itemlist = channel.search(item, tecleado) else: from core import support itemlist = support.search(channel, item, tecleado) else: return platformtools.render_items(itemlist, item) # For all other actions else: # import web_pdb; web_pdb.set_trace() logger.info("Executing channel '%s' method" % item.action) itemlist = getattr(channel, item.action)(item) if config.get_setting('trakt_sync'): from core import trakt_tools token_auth = config.get_setting("token_trakt", "trakt") if not token_auth: trakt_tools.auth_trakt() else: import xbmc if not xbmc.getCondVisibility('System.HasAddon(script.trakt)') and config.get_setting( 'install_trakt'): trakt_tools.ask_install_script() itemlist = trakt_tools.trakt_check(itemlist) else: config.set_setting('install_trakt', True) platformtools.render_items(itemlist, item) # except urllib2.URLError as e: # import traceback # logger.error(traceback.format_exc()) # # # Grab inner and third party errors # if hasattr(e, 'reason'): # logger.error("Reason for the error, code: %s | Reason: %s" % (str(e.reason[0]), str(e.reason[1]))) # texto = config.get_localized_string(30050) # "No se puede conectar con el sitio web" # platformtools.dialog_ok(config.get_localized_string(20000), texto) # # # Grab server response errors # elif hasattr(e, 'code'): # logger.error("HTTP error code: %d" % e.code) # # "El sitio web no funciona correctamente (error http %d)" # platformtools.dialog_ok(config.get_localized_string(20000), config.get_localized_string(30051) % e.code) except WebErrorException as e: import traceback from core import scrapertools logger.error(traceback.format_exc()) patron = 'File "' + os.path.join(config.get_runtime_path(), "channels", "").replace("\\", "\\\\") + '([^.]+)\.py"' canal = scrapertools.find_single_match(traceback.format_exc(), patron) platformtools.dialog_ok( config.get_localized_string(59985) + canal, config.get_localized_string(60013) %(e)) except: import traceback from core import scrapertools logger.error(traceback.format_exc()) patron = 'File "' + os.path.join(config.get_runtime_path(), "channels", "").replace("\\", "\\\\") + '([^.]+)\.py"' canal = scrapertools.find_single_match(traceback.format_exc(), patron) try: import xbmc if config.get_platform(True)['num_version'] < 14: log_name = "xbmc.log" else: log_name = "kodi.log" log_message = config.get_localized_string(50004) + xbmc.translatePath("special://logpath") + log_name except: log_message = "" if canal: if item.url: if platformtools.dialog_yesno(config.get_localized_string(60087) % canal, config.get_localized_string(60014), log_message, nolabel='ok', yeslabel=config.get_localized_string(70739)): run(Item(action="open_browser", url=item.url)) else: platformtools.dialog_ok(config.get_localized_string(60087) % canal, config.get_localized_string(60014), log_message) else: platformtools.dialog_ok( config.get_localized_string(60038), config.get_localized_string(60015), log_message)
def scrape(item, patron='', listGroups=[], headers="", blacklist="", data="", patron_block="", patronNext="", action="findvideos", url_host="", addVideolibrary=True): # patron: the patron to use for scraping page, all capturing group must match with listGroups # listGroups: a list containing the scraping info obtained by your patron, in order # accepted values are: url, title, thumb, quality, year, plot, duration, genre, rating # header: values to pass to request header # blacklist: titles that you want to exclude(service articles for example) # data: if you want to pass data manually, maybe because you need some custom replacement # patron_block: patron to get parts of the page (to scrape with patron attribute), # if you need a "block inside another block" you can create a list, please note that all matches # will be packed as string # patronNext: patron for scraping next page link # action: if you want results perform an action different from "findvideos", useful when scraping film by genres # url_host: string to prepend to scrapedurl, useful when url don't contain host # example usage: # import support # itemlist = [] # patron = 'blablabla' # headers = [['Referer', host]] # blacklist = 'Request a TV serie!' # return support.scrape(item, itemlist, patron, ['thumb', 'quality', 'url', 'title', 'year', 'plot'], # headers=headers, blacklist=blacklist) itemlist = [] if not data: data = httptools.downloadpage(item.url, headers=headers).data.replace("'", '"') data = re.sub('\n|\t', ' ', data) # replace all ' with " and eliminate newline, so we don't need to worry about log('DATA =', data) block = data if patron_block: if type(patron_block) == str: patron_block = [patron_block] for n, regex in enumerate(patron_block): blocks = scrapertoolsV2.find_multiple_matches(block, regex) block = "" for b in blocks: block += "\n" + b log('BLOCK ', n, '=', block) else: block = data if patron and listGroups: matches = scrapertoolsV2.find_multiple_matches(block, patron) log('MATCHES =', matches) known_keys = [ 'url', 'title', 'thumb', 'quality', 'year', 'plot', 'duration', 'genere', 'rating' ] for match in matches: if len(listGroups) > len(match): # to fix a bug match = list(match) match.extend([''] * (len(listGroups) - len(match))) scraped = {} for kk in known_keys: val = match[listGroups.index(kk)] if kk in listGroups else '' if kk == "url": val = url_host + val scraped[kk] = val title = scrapertoolsV2.decodeHtmlentities(scraped["title"]).strip() plot = scrapertoolsV2.htmlclean( scrapertoolsV2.decodeHtmlentities(scraped["plot"])) if scraped["quality"]: longtitle = '[B]' + title + '[/B] [COLOR blue][' + scraped[ "quality"] + '][/COLOR]' else: longtitle = '[B]' + title + '[/B]' if item.infoLabels[ "title"] or item.fulltitle: # if title is set, probably this is a list of episodes or video sources infolabels = item.infoLabels else: infolabels = {} if scraped["year"]: infolabels['year'] = scraped["year"] if scraped["plot"]: infolabels['plot'] = plot if scraped["duration"]: matches = scrapertoolsV2.find_multiple_matches( scraped["duration"], r'([0-9])\s*?(?:[hH]|:|\.|,|\\|\/|\||\s)\s*?([0-9]+)') for h, m in matches: scraped["duration"] = int(h) * 60 + int(m) if not matches: scraped["duration"] = scrapertoolsV2.find_single_match( scraped["duration"], r'(\d+)') infolabels['duration'] = int(scraped["duration"]) * 60 if scraped["genere"]: genres = scrapertoolsV2.find_multiple_matches( scraped["genere"], '[A-Za-z]+') infolabels['genere'] = ", ".join(genres) if scraped["rating"]: infolabels['rating'] = scrapertoolsV2.decodeHtmlentities( scraped["rating"]) if scraped["title"] not in blacklist: it = Item(channel=item.channel, action=action, contentType=item.contentType, title=longtitle, fulltitle=title, show=title, quality=scraped["quality"], url=scraped["url"], infoLabels=infolabels, thumbnail=scraped["thumb"]) for lg in list(set(listGroups).difference(known_keys)): it.__setattr__(lg, match[listGroups.index(lg)]) itemlist.append(it) if (item.contentType == "episode" and (action != "findvideos" and action != "play")) \ or (item.contentType == "movie" and action != "play"): tmdb.set_infoLabels_itemlist(itemlist, seekTmdb=True) else: for it in itemlist: it.infoLabels = item.infoLabels if patronNext: nextPage(itemlist, item, data, patronNext, 2) if addVideolibrary and (item.infoLabels["title"] or item.fulltitle): item.fulltitle = item.infoLabels["title"] videolibrary(itemlist, item) return itemlist