Example #1
0
def create_mainlist_item(data):
    
    category = data["category_dict"]
    data = data["data"]
    
    new_item = Item(channel=category["channel"], action="list_all", first=0,
        category= category["category"], thumbnail= category["thumbnail"])
    
    for key in data:
        new_item.__setattr__(key, data[key])
    
    return new_item
Example #2
0
def makeItem():
    logger.debug()
    if sys.argv[2]:
        sp = sys.argv[2].split('&')
        url = sp[0]
        item = Item().fromurl(url)
        if len(sp) > 1:
            for e in sp[1:]:
                key, val = e.split('=')
                if val.lower() == 'false': val = False
                elif val.lower() == 'true': val = True
                item.__setattr__(key, val)
    # If no item, this is mainlist
    else:
        item = Item(channel='channelselector', action='getmainlist', viewmode='movie')

    return item
Example #3
0
def run(item=None):
    # from core.support import dbg;dbg()
    logger.debug()
    if not item:
        # Extract item from sys.argv
        if sys.argv[2]:
            sp = sys.argv[2].split('&')
            url = sp[0]
            item = Item().fromurl(url)
            if len(sp) > 1:
                for e in sp[1:]:
                    key, val = e.split('=')
                    item.__setattr__(key, val)
        # If no item, this is mainlist
        else:
            item = Item(channel="channelselector",
                        action="getmainlist",
                        viewmode="movie")
        if not config.get_setting('show_once'):
            if not config.get_all_settings_addon():
                logger.error('corrupted settings.xml!!')
                settings_xml = os.path.join(config.get_data_path(),
                                            "settings.xml")
                settings_bak = os.path.join(config.get_data_path(),
                                            "settings.bak")
                if filetools.exists(settings_bak):
                    filetools.copy(settings_bak, settings_xml, True)
                    logger.info('restored settings.xml from backup')
                else:
                    filetools.write(settings_xml,
                                    '<settings version="2">\n</settings>'
                                    )  # resetted settings
            else:
                from platformcode import xbmc_videolibrary
                xbmc_videolibrary.ask_set_content(silent=False)
                config.set_setting('show_once', True)

    logger.info(item.tostring())

    try:
        if not config.get_setting('tmdb_active'):
            config.set_setting('tmdb_active', True)

        # If item has no action, stops here
        if item.action == "":
            logger.debug("Item without action")
            return

        # Action for main menu in channelselector
        elif item.action == "getmainlist":
            import channelselector

            itemlist = channelselector.getmainlist()

            platformtools.render_items(itemlist, item)

        # Action for channel types on channelselector: movies, series, etc.
        elif item.action == "getchanneltypes":
            import channelselector
            itemlist = channelselector.getchanneltypes()

            platformtools.render_items(itemlist, item)

        # Action for channel listing on channelselector
        elif item.action == "filterchannels":
            import channelselector
            itemlist = channelselector.filterchannels(item.channel_type)

            platformtools.render_items(itemlist, item)

        # Special action for playing a video from the library
        elif item.action == "play_from_library":
            play_from_library(item)
            return

        elif item.action == "keymap":
            from platformcode import keymaptools
            if item.open:
                return keymaptools.open_shortcut_menu()
            else:
                return keymaptools.set_key()

        elif item.channel == "infoplus":
            from platformcode import infoplus
            return infoplus.Main(item)

        elif item.channel == "backup":
            from platformcode import backup
            return getattr(backup, item.action)(item)

        elif item.channel == "elementum_download":
            from platformcode import elementum_download
            return getattr(elementum_download, item.action)(item)

        elif item.channel == "shortcuts":
            from platformcode import shortcuts
            return getattr(shortcuts, item.action)(item)

        elif item.channel == "autorenumber":
            from platformcode import autorenumber
            return getattr(autorenumber, item.action)(item)

        elif item.action == "delete_key":
            from platformcode import keymaptools
            return keymaptools.delete_key()

        elif item.action == "script":
            from core import tmdb
            if tmdb.drop_bd():
                platformtools.dialog_notification(
                    config.get_localized_string(20000),
                    config.get_localized_string(60011),
                    time=2000,
                    sound=False)
        elif item.action == "itemInfo":
            platformtools.dialog_textviewer('Item info', item.parent)
        elif item.action == "open_browser":
            import webbrowser
            if not webbrowser.open(item.url):
                import xbmc
                if xbmc.getCondVisibility(
                        'system.platform.linux') and xbmc.getCondVisibility(
                            'system.platform.android'):  # android
                    xbmc.executebuiltin(
                        'StartAndroidActivity("", "android.intent.action.VIEW", "", "%s")'
                        % (item.url))
                else:
                    try:
                        import urllib.request as urllib
                    except ImportError:
                        import urllib
                    short = urllib.urlopen(
                        'https://u.nu/api.php?action=shorturl&format=simple&url='
                        + item.url).read().decode('utf-8')
                    platformtools.dialog_ok(
                        config.get_localized_string(20000),
                        config.get_localized_string(70740) % short)
        # Action in certain channel specified in "action" and "channel" parameters
        elif item.action == "check_channels":
            from platformcode import checkhost
            checkhost.check_channels()
        else:
            # Checks if channel exists
            if os.path.isfile(
                    os.path.join(config.get_runtime_path(), 'channels',
                                 item.channel + ".py")):
                CHANNELS = 'channels'
            else:
                CHANNELS = 'specials'

            channel_file = os.path.join(config.get_runtime_path(), CHANNELS,
                                        item.channel + ".py")

            logger.debug("channel_file= " + channel_file + ' - ' + CHANNELS +
                         ' - ' + item.channel)

            channel = None

            if os.path.exists(channel_file):
                try:
                    channel = __import__('%s.%s' % (CHANNELS, item.channel),
                                         None, None,
                                         ['%s.%s' % (CHANNELS, item.channel)])
                except ImportError:
                    exec("import " + CHANNELS + "." + item.channel +
                         " as channel")

            logger.info("Running channel %s | %s" %
                        (channel.__name__, channel.__file__))

            # Special play action
            if item.action == "play":
                # define la info para trakt
                try:
                    from core import trakt_tools
                    trakt_tools.set_trakt_info(item)
                except:
                    pass
                logger.debug("item.action=%s" % item.action.upper())
                # logger.debug("item_toPlay: " + "\n" + item.tostring('\n'))

                # First checks if channel has a "play" function
                if hasattr(channel, 'play'):
                    logger.debug("Executing channel 'play' method")
                    itemlist = channel.play(item)
                    b_favourite = item.isFavourite
                    # Play should return a list of playable URLS
                    if len(itemlist) > 0 and isinstance(itemlist[0], Item):
                        item = itemlist[0]
                        if b_favourite:
                            item.isFavourite = True
                        platformtools.play_video(item)

                    # Permitir varias calidades desde play en el Channel
                    elif len(itemlist) > 0 and isinstance(itemlist[0], list):
                        item.video_urls = itemlist
                        platformtools.play_video(item)

                    # If not, shows user an error message
                    else:
                        platformtools.dialog_ok(
                            config.get_localized_string(20000),
                            config.get_localized_string(60339))

                # If player don't have a "play" function, not uses the standard play from platformtools
                else:
                    logger.debug("Executing core 'play' method")
                    platformtools.play_video(item)

            # Special action for findvideos, where the plugin looks for known urls
            elif item.action == "findvideos":
                from core import servertools

                # First checks if channel has a "findvideos" function
                if hasattr(channel, 'findvideos'):
                    itemlist = getattr(channel, item.action)(item)

                # If not, uses the generic findvideos function
                else:
                    logger.debug("No channel 'findvideos' method, "
                                 "executing core method")
                    itemlist = servertools.find_video_items(item)

                if config.get_setting("max_links", "videolibrary") != 0:
                    itemlist = limit_itemlist(itemlist)

                from platformcode import subtitletools
                subtitletools.saveSubtitleName(item)

                platformtools.render_items(itemlist, item)

            # Special action for adding a movie to the library
            elif item.action == "add_pelicula_to_library":
                from core import videolibrarytools
                videolibrarytools.add_movie(item)

            # Special action for adding a serie to the library
            elif item.action == "add_serie_to_library":
                from core import videolibrarytools
                videolibrarytools.add_tvshow(item, channel)

            # Special action for downloading all episodes from a serie
            elif item.action == "download_all_episodes":
                from specials import downloads
                item.action = item.extra
                del item.extra
                downloads.save_download(item)

            # Special action for searching, first asks for the words then call the "search" function
            elif item.action == "search":
                # from core.support import dbg;dbg()
                if filetools.isfile(temp_search_file) and config.get_setting(
                        'videolibrary_kodi'):
                    itemlist = []
                    f = filetools.read(temp_search_file)
                    strList = f.split(',')
                    if strList[0] == '[V]' and strList[1] == item.channel:
                        for it in strList:
                            if it and it not in ['[V]', item.channel]:
                                itemlist.append(Item().fromurl(it))
                        filetools.write(temp_search_file, f[4:])
                        return platformtools.render_items(itemlist, item)
                    else:
                        filetools.remove(temp_search_file)

                logger.debug("item.action=%s" % item.action.upper())
                from core import channeltools

                if config.get_setting('last_search'):
                    last_search = channeltools.get_channel_setting(
                        'Last_searched', 'search', '')
                else:
                    last_search = ''

                search_text = platformtools.dialog_input(last_search)

                if search_text is not None:
                    channeltools.set_channel_setting('Last_searched',
                                                     search_text, 'search')
                    itemlist = new_search(item.clone(text=search_text),
                                          channel)
                else:
                    return

                platformtools.render_items(itemlist, item)

            # For all other actions
            else:
                # import web_pdb; web_pdb.set_trace()
                logger.debug("Executing channel '%s' method" % item.action)
                itemlist = getattr(channel, item.action)(item)
                if config.get_setting('trakt_sync'):
                    from core import trakt_tools
                    token_auth = config.get_setting("token_trakt", "trakt")
                    if not token_auth:
                        trakt_tools.auth_trakt()
                    else:
                        import xbmc
                        if not xbmc.getCondVisibility(
                                'System.HasAddon(script.trakt)'
                        ) and config.get_setting('install_trakt'):
                            trakt_tools.ask_install_script()
                    itemlist = trakt_tools.trakt_check(itemlist)
                else:
                    config.set_setting('install_trakt', True)

                platformtools.render_items(itemlist, item)

    except WebErrorException as e:
        import traceback
        from core import scrapertools

        logger.error(traceback.format_exc())

        platformtools.dialog_ok(
            config.get_localized_string(59985) % e.channel,
            config.get_localized_string(60013) % e.url)
    except Exception as e:
        import traceback
        from core import scrapertools

        logger.error(traceback.format_exc())

        patron = 'File "' + os.path.join(config.get_runtime_path(), "channels",
                                         "").replace("\\",
                                                     "\\\\") + r'([^.]+)\.py"'
        Channel = scrapertools.find_single_match(traceback.format_exc(),
                                                 patron)

        if Channel or e.__class__ == logger.ChannelScraperException:
            if item.url:
                if platformtools.dialog_yesno(
                        config.get_localized_string(60087) % Channel,
                        config.get_localized_string(60014),
                        nolabel='ok',
                        yeslabel=config.get_localized_string(70739)):
                    run(Item(action="open_browser", url=item.url))
            else:
                platformtools.dialog_ok(
                    config.get_localized_string(60087) % Channel,
                    config.get_localized_string(60014))
        else:
            if platformtools.dialog_yesno(config.get_localized_string(60038),
                                          config.get_localized_string(60015)):
                run(Item(channel="setting", action="report_menu"))
Example #4
0
def run(item=None):
    logger.info()

    if not item:
        # Extract item from sys.argv
        if sys.argv[2]:
            sp = sys.argv[2].split('&')
            url = sp[0]
            item = Item().fromurl(url)
            if len(sp) > 1:
                for e in sp[1:]:
                    key, val = e.split('=')
                    item.__setattr__(key, val)

        # If no item, this is mainlist
        else:
            if config.get_setting("start_page"):

                if not config.get_setting("custom_start"):
                    category = config.get_setting("category").lower()
                    item = Item(channel="news", action="novedades", extra=category, mode = 'silent')
                else:
                    from channels import side_menu
                    item= Item()
                    item = side_menu.check_user_home(item)
                    item.start = True;
            else:
                item = Item(channel="channelselector", action="getmainlist", viewmode="movie")
        if not config.get_setting('show_once'):
            from platformcode import xbmc_videolibrary
            xbmc_videolibrary.ask_set_content(1)
            config.set_setting('show_once', True)

    logger.info(item.tostring())

    try:
        if not config.get_setting('tmdb_active'):
            config.set_setting('tmdb_active', True)

        # If item has no action, stops here
        if item.action == "":
            logger.info("Item sin accion")
            return

        # Action for main menu in channelselector
        elif item.action == "getmainlist":
            import channelselector

            itemlist = channelselector.getmainlist()

            platformtools.render_items(itemlist, item)

        # Action for channel types on channelselector: movies, series, etc.
        elif item.action == "getchanneltypes":
            import channelselector
            itemlist = channelselector.getchanneltypes()

            platformtools.render_items(itemlist, item)

        # Action for channel listing on channelselector
        elif item.action == "filterchannels":
            import channelselector
            itemlist = channelselector.filterchannels(item.channel_type)

            platformtools.render_items(itemlist, item)

        # Special action for playing a video from the library
        elif item.action == "play_from_library":
            play_from_library(item)
            return

        elif item.action == "keymap":
            from platformcode import keymaptools
            if item.open:
                return keymaptools.open_shortcut_menu()
            else:
                return keymaptools.set_key()

        elif item.action == "script":
            from core import tmdb
            if tmdb.drop_bd():
                platformtools.dialog_notification(config.get_localized_string(20000), config.get_localized_string(60011), time=2000, sound=False)

        # Action in certain channel specified in "action" and "channel" parameters
        else:

            # Entry point for a channel is the "mainlist" action, so here we check parental control
            if item.action == "mainlist":


                # Parental control
                # If it is an adult channel, and user has configured pin, asks for it
                if channeltools.is_adult(item.channel) and config.get_setting("adult_request_password"):
                    tecleado = platformtools.dialog_input("", config.get_localized_string(60334), True)
                    if tecleado is None or tecleado != config.get_setting("adult_password"):
                        return

            # # Actualiza el canal individual
            # if (item.action == "mainlist" and item.channel != "channelselector" and
            #             config.get_setting("check_for_channel_updates") == True):
            #     from core import updater
            #     updater.update_channel(item.channel)

            # Checks if channel exists
            channel_file = os.path.join(config.get_runtime_path(),
                                        'channels', item.channel + ".py")
            logger.info("channel_file=%s" % channel_file)

            channel = None

            if os.path.exists(channel_file):
                try:
                    channel = __import__('channels.%s' % item.channel, None,
                                         None, ["channels.%s" % item.channel])
                except ImportError:
                    exec("import channels." + item.channel + " as channel")

            logger.info("Running channel %s | %s" % (channel.__name__, channel.__file__))

            # Special play action
            if item.action == "play":
                #define la info para trakt
                try:
                    trakt_tools.set_trakt_info(item)
                except:
                    pass
                logger.info("item.action=%s" % item.action.upper())
                # logger.debug("item_toPlay: " + "\n" + item.tostring('\n'))

                # First checks if channel has a "play" function
                if hasattr(channel, 'play'):
                    logger.info("Executing channel 'play' method")
                    itemlist = channel.play(item)
                    b_favourite = item.isFavourite
                    # Play should return a list of playable URLS
                    if len(itemlist) > 0 and isinstance(itemlist[0], Item):
                        item = itemlist[0]
                        if b_favourite:
                            item.isFavourite = True
                        platformtools.play_video(item)

                    # Permitir varias calidades desde play en el canal
                    elif len(itemlist) > 0 and isinstance(itemlist[0], list):
                        item.video_urls = itemlist
                        platformtools.play_video(item)

                    # If not, shows user an error message
                    else:
                        platformtools.dialog_ok(config.get_localized_string(20000), config.get_localized_string(60339))

                # If player don't have a "play" function, not uses the standard play from platformtools
                else:
                    logger.info("Executing core 'play' method")
                    platformtools.play_video(item)

            # Special action for findvideos, where the plugin looks for known urls
            elif item.action == "findvideos":

                # First checks if channel has a "findvideos" function
                if hasattr(channel, 'findvideos'):
                    itemlist = getattr(channel, item.action)(item)
                    itemlist = servertools.filter_servers(itemlist)

                # If not, uses the generic findvideos function
                else:
                    logger.info("No channel 'findvideos' method, "
                                "executing core method")
                    itemlist = servertools.find_video_items(item)

                if config.get_setting("max_links", "videolibrary") != 0:
                    itemlist = limit_itemlist(itemlist)

                from platformcode import subtitletools
                subtitletools.saveSubtitleName(item)

                platformtools.render_items(itemlist, item)

            # Special action for adding a movie to the library
            elif item.action == "add_pelicula_to_library":
                videolibrarytools.add_movie(item)

            # Special action for adding a serie to the library
            elif item.action == "add_serie_to_library":
                videolibrarytools.add_tvshow(item, channel)

            # Special action for downloading all episodes from a serie
            elif item.action == "download_all_episodes":
                from channels import downloads
                item.action = item.extra
                del item.extra
                downloads.save_download(item)

            # Special action for searching, first asks for the words then call the "search" function
            elif item.action == "search":
                logger.info("item.action=%s" % item.action.upper())

                # last_search = ""
                # last_search_active = config.get_setting("last_search", "search")
                # if last_search_active:
                #     try:
                #         current_saved_searches_list = list(config.get_setting("saved_searches_list", "search"))
                #         last_search = current_saved_searches_list[0]
                #     except:
                #         pass

                last_search = channeltools.get_channel_setting('Last_searched', 'search', '')

                tecleado = platformtools.dialog_input(last_search)

                if tecleado is not None:
                    channeltools.set_channel_setting('Last_searched', tecleado, 'search')
                    itemlist = channel.search(item, tecleado)
                else:
                    return

                platformtools.render_items(itemlist, item)

            # For all other actions
            else:
                logger.info("Executing channel '%s' method" % item.action)
                itemlist = getattr(channel, item.action)(item)
                if config.get_setting('trakt_sync'):
                    token_auth = config.get_setting("token_trakt", "trakt")
                    if not token_auth:
                        trakt_tools.auth_trakt()
                    else:
                        import xbmc
                        if not xbmc.getCondVisibility('System.HasAddon(script.trakt)') and config.get_setting(
                                'install_trakt'):
                            trakt_tools.ask_install_script()
                    itemlist = trakt_tools.trakt_check(itemlist)
                else:
                    config.set_setting('install_trakt', True)

                platformtools.render_items(itemlist, item)

    except urllib2.URLError as e:
        import traceback
        logger.error(traceback.format_exc())

        # Grab inner and third party errors
        if hasattr(e, 'reason'):
            logger.error("Razon del error, codigo: %s | Razon: %s" % (str(e.reason[0]), str(e.reason[1])))
            texto = config.get_localized_string(30050)  # "No se puede conectar con el sitio web"
            platformtools.dialog_ok("alfa", texto)

        # Grab server response errors
        elif hasattr(e, 'code'):
            logger.error("Codigo de error HTTP : %d" % e.code)
            # "El sitio web no funciona correctamente (error http %d)"
            platformtools.dialog_ok("alfa", config.get_localized_string(30051) % e.code)
    except WebErrorException as e:
        import traceback
        logger.error(traceback.format_exc())

        patron = 'File "' + os.path.join(config.get_runtime_path(), "channels", "").replace("\\",
                                                                                            "\\\\") + '([^.]+)\.py"'
        canal = scrapertools.find_single_match(traceback.format_exc(), patron)

        platformtools.dialog_ok(
            config.get_localized_string(59985) + canal,
            config.get_localized_string(60013) %(e))
    except:
        import traceback
        logger.error(traceback.format_exc())

        patron = 'File "' + os.path.join(config.get_runtime_path(), "channels", "").replace("\\",
                                                                                            "\\\\") + '([^.]+)\.py"'
        canal = scrapertools.find_single_match(traceback.format_exc(), patron)

        try:
            import xbmc
            if config.get_platform(True)['num_version'] < 14:
                log_name = "xbmc.log"
            else:
                log_name = "kodi.log"
            log_message = config.get_localized_string(50004) + xbmc.translatePath("special://logpath") + log_name
        except:
            log_message = ""

        if canal:
            platformtools.dialog_ok(
                config.get_localized_string(60087) %canal,
                config.get_localized_string(60014),
                log_message)
        else:
            platformtools.dialog_ok(
                config.get_localized_string(60038),
                config.get_localized_string(60015),
                log_message)
Example #5
0
def scrapeBlock(item, args, block, patron, headers, action, pagination, debug,
                typeContentDict, typeActionDict, blacklist, search, pag,
                function, lang):
    itemlist = []
    log("scrapeBlock qui")
    matches = scrapertools.find_multiple_matches_groups(block, patron)
    log('MATCHES =', matches)

    if debug:
        regexDbg(item, patron, headers, block)

    known_keys = [
        'url', 'title', 'title2', 'season', 'episode', 'thumb', 'quality',
        'year', 'plot', 'duration', 'genere', 'rating', 'type', 'lang'
    ]
    # Legenda known_keys per i groups nei patron
    # known_keys = ['url', 'title', 'title2', 'season', 'episode', 'thumb', 'quality',
    #                'year', 'plot', 'duration', 'genere', 'rating', 'type', 'lang']
    # url = link relativo o assoluto alla pagina titolo film/serie
    # title = titolo Film/Serie/Anime/Altro
    # title2 = titolo dell'episodio Serie/Anime/Altro
    # season = stagione in formato numerico
    # episode = numero episodio, in formato numerico.
    # thumb = linkrealtivo o assoluto alla locandina Film/Serie/Anime/Altro
    # quality = qualità indicata del video
    # year = anno in formato numerico (4 cifre)
    # duration = durata del Film/Serie/Anime/Altro
    # genere = genere del Film/Serie/Anime/Altro. Es: avventura, commedia
    # rating = punteggio/voto in formato numerico
    # type = tipo del video. Es. movie per film o tvshow per le serie. Di solito sono discrimanti usati dal sito
    # lang = lingua del video. Es: ITA, Sub-ITA, Sub, SUB ITA.
    # AVVERTENZE: Se il titolo è trovato nella ricerca TMDB/TVDB/Altro allora le locandine e altre info non saranno quelle recuperate nel sito.!!!!

    stagione = ''  # per quei siti che hanno la stagione nel blocco ma non nelle puntate
    for i, match in enumerate(matches):
        if pagination and (pag - 1) * pagination > i: continue  # pagination
        if pagination and i >= pag * pagination: break  # pagination
        listGroups = match.keys()
        match = match.values()

        if len(listGroups) > len(match):  # to fix a bug
            match = list(match)
            match.extend([''] * (len(listGroups) - len(match)))

        scraped = {}
        for kk in known_keys:
            val = match[listGroups.index(kk)] if kk in listGroups else ''
            if val and (kk == "url" or kk == 'thumb') and 'http' not in val:
                val = scrapertools.find_single_match(
                    item.url, 'https?://[a-z0-9.-]+') + val
            scraped[kk] = val

        if scraped['season']:
            stagione = scraped['season']
            episode = scraped['season'] + 'x' + scraped['episode']
        elif item.season:
            episode = item.season + 'x' + scraped['episode']
        elif item.contentType == 'tvshow' and (scraped['episode'] == ''
                                               and scraped['season'] == ''
                                               and stagione == ''):
            item.news = 'season_completed'
            episode = ''
        else:
            episode = re.sub(r'\s-\s|-|x|&#8211|&#215;|×', 'x',
                             scraped['episode']) if scraped['episode'] else ''
            second_episode = scrapertools.find_single_match(
                episode, 'x\d+x(\d+)')
            if second_episode:
                episode = re.sub(r'(\d+x\d+)x\d+', r'\1-',
                                 episode) + second_episode.zfill(2)

        #episode = re.sub(r'\s-\s|-|x|&#8211|&#215;', 'x', scraped['episode']) if scraped['episode'] else ''
        title = cleantitle(scraped['title']) if scraped['title'] else ''
        title2 = cleantitle(scraped['title2']) if scraped['title2'] else ''
        quality = scraped['quality'].strip() if scraped['quality'] else ''
        Type = scraped['type'] if scraped['type'] else ''
        plot = cleantitle(scraped["plot"]) if scraped["plot"] else ''

        # make formatted Title [longtitle]
        s = ' - '
        title = episode + (s if episode and title else '') + title
        longtitle = title + (s if title and title2 else '') + title2
        longtitle = typo(longtitle, 'bold')
        longtitle += typo(quality, '_ [] color kod') if quality else ''

        lang1, longtitle = scrapeLang(scraped, lang, longtitle)

        # if title is set, probably this is a list of episodes or video sources
        # necessaria l'aggiunta di == scraped["title"] altrimenti non prende i gruppi dopo le categorie
        if item.infoLabels["title"] == scraped["title"]:
            infolabels = item.infoLabels
        else:
            infolabels = {}
            if scraped['year']:
                infolabels['year'] = scraped['year']
            if scraped["plot"]:
                infolabels['plot'] = plot
            if scraped['duration']:
                matches = scrapertools.find_multiple_matches(
                    scraped['duration'],
                    r'([0-9])\s*?(?:[hH]|:|\.|,|\\|\/|\||\s)\s*?([0-9]+)')
                for h, m in matches:
                    scraped['duration'] = int(h) * 60 + int(m)
                if not matches:
                    scraped['duration'] = scrapertools.find_single_match(
                        scraped['duration'], r'(\d+)')
                infolabels['duration'] = int(scraped['duration']) * 60
            if scraped['genere']:
                genres = scrapertools.find_multiple_matches(
                    scraped['genere'], '[A-Za-z]+')
                infolabels['genere'] = ", ".join(genres)
            if scraped["rating"]:
                infolabels['rating'] = scrapertools.decodeHtmlentities(
                    scraped["rating"])

        AC = CT = ''
        if typeContentDict:
            for name, variants in typeContentDict.items():
                if str(scraped['type']).lower() in variants:
                    CT = name
                    break
                else:
                    CT = item.contentType
        if typeActionDict:
            for name, variants in typeActionDict.items():
                if str(scraped['type']).lower() in variants:
                    AC = name
                    break
                else:
                    AC = action

        if (scraped["title"] not in blacklist) and (search.lower()
                                                    in longtitle.lower()):
            it = Item(
                channel=item.channel,
                action=AC if AC else action,
                contentType='episode'
                if function == 'episodios' else CT if CT else item.contentType,
                title=longtitle,
                fulltitle=item.fulltitle if function == 'episodios' else title,
                show=item.show if function == 'episodios' else title,
                quality=quality,
                url=scraped["url"],
                infoLabels=infolabels,
                thumbnail=item.thumbnail
                if function == 'episodios' else scraped["thumb"],
                args=item.args,
                contentSerieName=scraped['title'] if item.contentType
                or CT != 'movie' and function != 'episodios' else
                item.fulltitle if function == 'episodios' else '',
                contentTitle=scraped['title']
                if item.contentType or CT == 'movie' else '',
                contentLanguage=lang1,
                contentEpisodeNumber=episode if episode else '',
                news=item.news if item.news else '')

            for lg in list(set(listGroups).difference(known_keys)):
                it.__setattr__(lg, match[listGroups.index(lg)])

            if 'itemHook' in args:
                it = args['itemHook'](it)
            itemlist.append(it)

    return itemlist, matches
Example #6
0
def scrapeBlock(item, args, block, patron, headers, action, pagination, debug,
                typeContentDict, typeActionDict, blacklist, search, pag,
                function, lang):
    itemlist = []
    log("scrapeBlock qui", block, patron)
    matches = scrapertoolsV2.find_multiple_matches_groups(block, patron)
    log('MATCHES =', matches)

    if debug:
        regexDbg(item, patron, headers, block)

    known_keys = [
        'url', 'title', 'title2', 'episode', 'thumb', 'quality', 'year',
        'plot', 'duration', 'genere', 'rating', 'type', 'lang'
    ]
    # lang = ''  # aggiunto per gestire i siti con pagine di serietv dove si hanno i video in ita e in subita
    for i, match in enumerate(matches):
        if pagination and (pag - 1) * pagination > i: continue  # pagination
        if pagination and i >= pag * pagination: break  # pagination
        listGroups = match.keys()
        match = match.values()

        if len(listGroups) > len(match):  # to fix a bug
            match = list(match)
            match.extend([''] * (len(listGroups) - len(match)))

        scraped = {}
        for kk in known_keys:
            val = match[listGroups.index(kk)] if kk in listGroups else ''
            if val and (kk == "url" or kk == 'thumb') and 'http' not in val:
                val = scrapertoolsV2.find_single_match(
                    item.url, 'https?://[a-z0-9.-]+') + val
            scraped[kk] = val

        episode = re.sub(r'\s-\s|-|x|&#8211|&#215;', 'x',
                         scraped['episode']) if scraped['episode'] else ''
        title = cleantitle(scraped['title']) if scraped['title'] else ''
        title2 = cleantitle(scraped['title2']) if scraped['title2'] else ''
        quality = scraped['quality'].strip() if scraped['quality'] else ''
        Type = scraped['type'] if scraped['type'] else ''
        plot = cleantitle(scraped["plot"]) if scraped["plot"] else ''

        # make formatted Title [longtitle]
        s = ' - '
        title = episode + (s if episode and title else '') + title
        longtitle = title + (s if title and title2 else '') + title2
        longtitle = typo(longtitle, 'bold')
        longtitle += (typo(Type, '_ () bold') if Type else '') + (typo(
            quality, '_ [] color kod') if quality else '')

        # # per togliere la voce [ITA] da liste che non siano titoli (es.: genere)
        # if action != 'peliculas':
        #     lang, longtitle = scrapeLang(scraped, lang, longtitle)
        # else:
        #     longtitle = longtitle.replace('[ITA]','')
        #     lang = ''

        lang, longtitle = scrapeLang(scraped, lang, longtitle)

        # if title is set, probably this is a list of episodes or video sources
        # necessaria l'aggiunta di == scraped["title"] altrimenti non prende i gruppi dopo le categorie
        if item.infoLabels["title"] == scraped["title"]:
            infolabels = item.infoLabels
        else:
            infolabels = {}
            if scraped['year']:
                infolabels['year'] = scraped['year']
            if scraped["plot"]:
                infolabels['plot'] = plot
            if scraped['duration']:
                matches = scrapertoolsV2.find_multiple_matches(
                    scraped['duration'],
                    r'([0-9])\s*?(?:[hH]|:|\.|,|\\|\/|\||\s)\s*?([0-9]+)')
                for h, m in matches:
                    scraped['duration'] = int(h) * 60 + int(m)
                if not matches:
                    scraped['duration'] = scrapertoolsV2.find_single_match(
                        scraped['duration'], r'(\d+)')
                infolabels['duration'] = int(scraped['duration']) * 60
            if scraped['genere']:
                genres = scrapertoolsV2.find_multiple_matches(
                    scraped['genere'], '[A-Za-z]+')
                infolabels['genere'] = ", ".join(genres)
            if scraped["rating"]:
                infolabels['rating'] = scrapertoolsV2.decodeHtmlentities(
                    scraped["rating"])

        AC = CT = ''
        if typeContentDict:
            for name, variants in typeContentDict.items():
                if str(scraped['type']).lower() in variants:
                    CT = name
                else:
                    CT = item.contentType
        if typeActionDict:
            for name, variants in typeActionDict.items():
                if str(scraped['type']).lower() in variants:
                    AC = name
                else:
                    AC = action

        if (scraped["title"] not in blacklist) and (search.lower()
                                                    in longtitle.lower()):
            it = Item(
                channel=item.channel,
                action=AC if AC else action,
                contentType='episode'
                if function == 'episodios' else CT if CT else item.contentType,
                title=longtitle,
                fulltitle=item.fulltitle if function == 'episodios' else title,
                show=item.show if function == 'episodios' else title,
                quality=quality,
                url=scraped["url"],
                infoLabels=infolabels,
                thumbnail=item.thumbnail
                if function == 'episodios' else scraped["thumb"],
                args=item.args,
                contentSerieName=title
                if item.contentType != 'movie' and function != 'episodios' else
                item.fulltitle if function == 'episodios' else '',
                contentTitle=title if item.contentType == 'movie' else '',
                contentLanguage=lang,
                ep=episode if episode else '')

            for lg in list(set(listGroups).difference(known_keys)):
                it.__setattr__(lg, match[listGroups.index(lg)])

            if 'itemHook' in args:
                it = args['itemHook'](it)
            itemlist.append(it)

    return itemlist, matches
Example #7
0
def scrape(item,
           patron='',
           listGroups=[],
           headers="",
           blacklist="",
           data="",
           patron_block="",
           patronNext="",
           action="findvideos",
           addVideolibrary=True,
           type_content_dict={},
           type_action_dict={}):
    # patron: the patron to use for scraping page, all capturing group must match with listGroups
    # listGroups: a list containing the scraping info obtained by your patron, in order
    # accepted values are: url, title, thumb, quality, year, plot, duration, genre, rating, episode, lang

    # header: values to pass to request header
    # blacklist: titles that you want to exclude(service articles for example)
    # data: if you want to pass data manually, maybe because you need some custom replacement
    # patron_block: patron to get parts of the page (to scrape with patron attribute),
    #               if you need a "block inside another block" you can create a list, please note that all matches
    #               will be packed as string
    # patronNext: patron for scraping next page link
    # action: if you want results perform an action different from "findvideos", useful when scraping film by genres
    # url_host: string to prepend to scrapedurl, useful when url don't contain host
    # example usage:
    #   import support
    #   itemlist = []
    #   patron = 'blablabla'
    #   headers = [['Referer', host]]
    #   blacklist = 'Request a TV serie!'
    #   return support.scrape(item, itemlist, patron, ['thumb', 'quality', 'url', 'title', 'title2', 'year', 'plot', 'episode', 'lang'],
    #                           headers=headers, blacklist=blacklist)
    # listGroups
    #    thumb = immagine, quality = qualità, url = link singolo o gruppo, title = titolo film o serie, title2 = titolo aggiuntivo
    #    year = anno del film o della serie, plot = descrizione film o serie, episode = numero stagione - numero episodio in caso di serie,
    #    lang = lingua del video
    # 'type' is a check for typologies of content e.g. Film or TV Series
    # 'episode' is a key to grab episode numbers if it is separated from the title
    # IMPORTANT 'type' is a special key, to work need type_content_dict={} and type_action_dict={}

    itemlist = []

    if not data:
        data = httptools.downloadpage(item.url,
                                      headers=headers,
                                      ignore_response_code=True).data.replace(
                                          "'", '"')
        data = re.sub('\n|\t', ' ', data)
        # replace all ' with " and eliminate newline, so we don't need to worry about
        log('DATA =', data)

        block = data

        if patron_block:
            if type(patron_block) == str:
                patron_block = [patron_block]

            for n, regex in enumerate(patron_block):
                blocks = scrapertoolsV2.find_multiple_matches(block, regex)
                block = ""
                for b in blocks:
                    block += "\n" + str(b)
                log('BLOCK ', n, '=', block)
    else:
        block = data
    if patron and listGroups:
        matches = scrapertoolsV2.find_multiple_matches(block, patron)
        log('MATCHES =', matches)

        known_keys = [
            'url', 'title', 'title2', 'episode', 'thumb', 'quality', 'year',
            'plot', 'duration', 'genere', 'rating', 'type', 'lang'
        ]  #by greko aggiunto episode
        lang = ''  # aggiunto per gestire i siti con pagine di serietv dove si hanno i video in ita e in subita

        for match in matches:
            if len(listGroups) > len(match):  # to fix a bug
                match = list(match)
                match.extend([''] * (len(listGroups) - len(match)))

            scraped = {}
            for kk in known_keys:
                val = match[listGroups.index(kk)] if kk in listGroups else ''
                if val and (kk == "url"
                            or kk == 'thumb') and 'http' not in val:
                    val = scrapertoolsV2.find_single_match(
                        item.url, 'https?://[a-z0-9.-]+') + val
                scraped[kk] = val

            title = scrapertoolsV2.htmlclean(
                scrapertoolsV2.decodeHtmlentities(scraped["title"])).replace(
                    '’', '\'').replace('"',
                                       "'").strip()  # fix by greko da " a '
            plot = scrapertoolsV2.htmlclean(
                scrapertoolsV2.decodeHtmlentities(scraped["plot"]))

            longtitle = typo(title, 'bold')
            if scraped['quality']:
                longtitle = longtitle + typo(scraped['quality'],
                                             '_ [] color kod')
            if scraped['episode']:
                scraped['episode'] = re.sub(r'\s-\s|-|x|&#8211', 'x',
                                            scraped['episode'])
                longtitle = typo(scraped['episode'] + ' - ',
                                 'bold') + longtitle
            if scraped['title2']:
                title2 = scrapertoolsV2.htmlclean(
                    scrapertoolsV2.decodeHtmlentities(
                        scraped["title2"])).replace('"', "'").strip()
                longtitle = longtitle + typo(title2, 'bold _ -- _')

            ##    Aggiunto/modificato per gestire i siti che hanno i video
            ##    in ita e subita delle serie tv nella stessa pagina
            if scraped['lang']:
                if 'sub' in scraped['lang'].lower():
                    lang = 'Sub-ITA'
                else:
                    lang = 'ITA'
            if lang != '':
                longtitle += typo(lang, '_ [] color kod')

            if item.infoLabels[
                    "title"] or item.fulltitle:  # if title is set, probably this is a list of episodes or video sources
                infolabels = item.infoLabels
            else:
                infolabels = {}
                if scraped["year"]:
                    infolabels['year'] = scraped["year"]
                if scraped["plot"]:
                    infolabels['plot'] = plot
                if scraped["duration"]:
                    matches = scrapertoolsV2.find_multiple_matches(
                        scraped["duration"],
                        r'([0-9])\s*?(?:[hH]|:|\.|,|\\|\/|\||\s)\s*?([0-9]+)')
                    for h, m in matches:
                        scraped["duration"] = int(h) * 60 + int(m)
                    if not matches:
                        scraped["duration"] = scrapertoolsV2.find_single_match(
                            scraped["duration"], r'(\d+)')
                    infolabels['duration'] = int(scraped["duration"]) * 60
                if scraped["genere"]:
                    genres = scrapertoolsV2.find_multiple_matches(
                        scraped["genere"], '[A-Za-z]+')
                    infolabels['genere'] = ", ".join(genres)
                if scraped["rating"]:
                    infolabels['rating'] = scrapertoolsV2.decodeHtmlentities(
                        scraped["rating"])

            if type_content_dict:
                for name, variants in type_content_dict.items():
                    if scraped['type'] in variants:
                        item.contentType = name
            if type_action_dict:
                for name, variants in type_action_dict.items():
                    if scraped['type'] in variants:
                        action = name

            if inspect.stack()[1][3] == 'episodios':
                item.contentType = 'episode'

            if scraped["title"] not in blacklist:
                it = Item(channel=item.channel,
                          action=action,
                          contentType=item.contentType,
                          title=longtitle,
                          fulltitle=title,
                          show=title,
                          language=lang if lang != '' else '',
                          quality=scraped["quality"],
                          url=scraped["url"],
                          infoLabels=infolabels,
                          thumbnail=scraped["thumb"],
                          args=item.args)

                for lg in list(set(listGroups).difference(known_keys)):
                    it.__setattr__(lg, match[listGroups.index(lg)])

                itemlist.append(it)
        checkHost(item, itemlist)
        if (item.contentType == "tvshow" and (action != "findvideos" and action != "play")) \
                or (item.contentType == "episode" and action != "play") \
                or (item.contentType == "movie" and action != "play"):
            tmdb.set_infoLabels_itemlist(itemlist, seekTmdb=True)
        else:
            for it in itemlist:
                it.infoLabels = item.infoLabels

        if patronNext:
            nextPage(itemlist, item, data, patronNext, 2)

        if addVideolibrary and (item.infoLabels["title"] or item.fulltitle):
            item.fulltitle = item.infoLabels["title"]
            videolibrary(itemlist, item)

    return itemlist
Example #8
0
def run(item=None):
    logger.info()
    if not item:
        # Extract item from sys.argv
        if sys.argv[2]:
            sp = sys.argv[2].split('&')
            url = sp[0]
            item = Item().fromurl(url)
            if len(sp) > 1:
                for e in sp[1:]:
                    key, val = e.split('=')
                    item.__setattr__(key, val)
        # If no item, this is mainlist
        else:
            if config.get_setting("start_page"):

                if not config.get_setting("custom_start"):
                    dictCategory = {
                        config.get_localized_string(70137): 'peliculas',
                        config.get_localized_string(30123): 'series',
                        config.get_localized_string(30124): 'anime',
                        config.get_localized_string(70018): 'infantiles',
                        config.get_localized_string(60513): 'documentales',
                        config.get_localized_string(70013): 'terror',
                        config.get_localized_string(70014): 'castellano',
                        config.get_localized_string(59976): 'latino',
                        config.get_localized_string(70171): 'torrent',
                    }
                    if not config.get_setting("category") in dictCategory.keys():
                        config.set_setting('category', config.get_localized_string(70137))
                    category = dictCategory[config.get_setting("category")]
                    item = Item(channel="news", action="novedades", extra=category, mode = 'silent')
                else:
                    from specials import side_menu
                    item= Item()
                    item = side_menu.check_user_home(item)
                    item.start = True
            else:
                item = Item(channel="channelselector", action="getmainlist", viewmode="movie")
        if not config.get_setting('show_once'):
            from platformcode import xbmc_videolibrary
            xbmc_videolibrary.ask_set_content(silent=False)
            config.set_setting('show_once', True)

    logger.info(item.tostring())

    try:
        if not config.get_setting('tmdb_active'):
            config.set_setting('tmdb_active', True)

        # If item has no action, stops here
        if item.action == "":
            logger.info("Item without action")
            return

        # Action for main menu in channelselector
        elif item.action == "getmainlist":
            import channelselector

            itemlist = channelselector.getmainlist()

            platformtools.render_items(itemlist, item)

        # Action for channel types on channelselector: movies, series, etc.
        elif item.action == "getchanneltypes":
            import channelselector
            itemlist = channelselector.getchanneltypes()

            platformtools.render_items(itemlist, item)

        # Action for channel listing on channelselector
        elif item.action == "filterchannels":
            import channelselector
            itemlist = channelselector.filterchannels(item.channel_type)

            platformtools.render_items(itemlist, item)

        # Special action for playing a video from the library
        elif item.action == "play_from_library":
            play_from_library(item)
            return

        elif item.action == "keymap":
            from platformcode import keymaptools
            if item.open:
                return keymaptools.open_shortcut_menu()
            else:
                return keymaptools.set_key()

        elif item.action == "delete_key":
            from platformcode import keymaptools
            return keymaptools.delete_key()

        elif item.action == "script":
            from core import tmdb
            if tmdb.drop_bd():
                platformtools.dialog_notification(config.get_localized_string(20000), config.get_localized_string(60011), time=2000, sound=False)
        elif item.action == "itemInfo":
            import base64
            platformtools.dialog_textviewer('Item info', item.parent)
        elif item.action == "open_browser":
            import webbrowser
            if not webbrowser.open(item.url):
                import xbmc
                if xbmc.getCondVisibility('system.platform.linux') and xbmc.getCondVisibility('system.platform.android'):  # android
                    xbmc.executebuiltin('StartAndroidActivity("", "android.intent.action.VIEW", "", "%s")' % (item.url))
                else:
                    try:
                        import urllib.request as urllib
                    except ImportError:
                        import urllib
                    short = urllib.urlopen(
                        'https://u.nu/api.php?action=shorturl&format=simple&url=' + item.url).read().decode('utf-8')
                    platformtools.dialog_ok(config.get_localized_string(20000),
                                            config.get_localized_string(70740) % short)
        # Action in certain channel specified in "action" and "channel" parameters
        else:
            # # Actualiza el canal individual
            # if (item.action == "mainlist" and item.channel != "channelselector" and
            #             config.get_setting("check_for_channel_updates") == True):
            #     from core import updater
            #     updater.update_channel(item.channel)

            # Checks if channel exists
            if os.path.isfile(os.path.join(config.get_runtime_path(), 'channels', item.channel + ".py")):
                CHANNELS = 'channels'
            else:
                CHANNELS = 'specials'

            channel_file = os.path.join(config.get_runtime_path(), CHANNELS, item.channel + ".py")

            logger.info("channel_file= " + channel_file + ' - ' + CHANNELS + ' - ' + item.channel)

            channel = None

            if os.path.exists(channel_file):
                try:
                    channel = __import__('%s.%s' % (CHANNELS, item.channel), None,
                                         None, ['%s.%s' % (CHANNELS, item.channel)])
                except ImportError:
                    exec("import " + CHANNELS + "." + item.channel + " as channel")

            logger.info("Running channel %s | %s" % (channel.__name__, channel.__file__))

            # Special play action
            if item.action == "play":
                #define la info para trakt
                try:
                    from core import trakt_tools
                    trakt_tools.set_trakt_info(item)
                except:
                    pass
                logger.info("item.action=%s" % item.action.upper())
                # logger.debug("item_toPlay: " + "\n" + item.tostring('\n'))

                # First checks if channel has a "play" function
                if hasattr(channel, 'play'):
                    logger.info("Executing channel 'play' method")
                    itemlist = channel.play(item)
                    b_favourite = item.isFavourite
                    # Play should return a list of playable URLS
                    if len(itemlist) > 0 and isinstance(itemlist[0], Item):
                        item = itemlist[0]
                        if b_favourite:
                            item.isFavourite = True
                        platformtools.play_video(item)

                    # Permitir varias calidades desde play en el canal
                    elif len(itemlist) > 0 and isinstance(itemlist[0], list):
                        item.video_urls = itemlist
                        platformtools.play_video(item)

                    # If not, shows user an error message
                    else:
                        platformtools.dialog_ok(config.get_localized_string(20000), config.get_localized_string(60339))

                # If player don't have a "play" function, not uses the standard play from platformtools
                else:
                    logger.info("Executing core 'play' method")
                    platformtools.play_video(item)

            # Special action for findvideos, where the plugin looks for known urls
            elif item.action == "findvideos":
                from core import servertools

                # First checks if channel has a "findvideos" function
                if hasattr(channel, 'findvideos'):
                    itemlist = getattr(channel, item.action)(item)
                    itemlist = servertools.filter_servers(itemlist)

                # If not, uses the generic findvideos function
                else:
                    logger.info("No channel 'findvideos' method, "
                                "executing core method")
                    itemlist = servertools.find_video_items(item)

                if config.get_setting("max_links", "videolibrary") != 0:
                    itemlist = limit_itemlist(itemlist)

                from platformcode import subtitletools
                subtitletools.saveSubtitleName(item)

                platformtools.render_items(itemlist, item)

            # Special action for adding a movie to the library
            elif item.action == "add_pelicula_to_library":
                from core import videolibrarytools
                videolibrarytools.add_movie(item)

            # Special action for adding a serie to the library
            elif item.action == "add_serie_to_library":
                from core import videolibrarytools
                videolibrarytools.add_tvshow(item, channel)

            # Special action for downloading all episodes from a serie
            elif item.action == "download_all_episodes":
                from specials import downloads
                item.action = item.extra
                del item.extra
                downloads.save_download(item)

            # Special action for searching, first asks for the words then call the "search" function
            elif item.action == "search":
                logger.info("item.action=%s" % item.action.upper())
                from core import channeltools

                if config.get_setting('last_search'):
                    last_search = channeltools.get_channel_setting('Last_searched', 'search', '')
                else:
                    last_search = ''

                tecleado = platformtools.dialog_input(last_search)

                if tecleado is not None:
                    channeltools.set_channel_setting('Last_searched', tecleado, 'search')
                    if 'search' in dir(channel):
                        itemlist = channel.search(item, tecleado)
                    else:
                        from core import support
                        itemlist = support.search(channel, item, tecleado)
                else:
                    return

                platformtools.render_items(itemlist, item)

            # For all other actions
            else:
                # import web_pdb; web_pdb.set_trace()
                logger.info("Executing channel '%s' method" % item.action)
                itemlist = getattr(channel, item.action)(item)
                if config.get_setting('trakt_sync'):
                    from core import trakt_tools
                    token_auth = config.get_setting("token_trakt", "trakt")
                    if not token_auth:
                        trakt_tools.auth_trakt()
                    else:
                        import xbmc
                        if not xbmc.getCondVisibility('System.HasAddon(script.trakt)') and config.get_setting(
                                'install_trakt'):
                            trakt_tools.ask_install_script()
                    itemlist = trakt_tools.trakt_check(itemlist)
                else:
                    config.set_setting('install_trakt', True)

                platformtools.render_items(itemlist, item)

    # except urllib2.URLError as e:
    #     import traceback
    #     logger.error(traceback.format_exc())
    #
    #     # Grab inner and third party errors
    #     if hasattr(e, 'reason'):
    #         logger.error("Reason for the error, code: %s | Reason: %s" % (str(e.reason[0]), str(e.reason[1])))
    #         texto = config.get_localized_string(30050)  # "No se puede conectar con el sitio web"
    #         platformtools.dialog_ok(config.get_localized_string(20000), texto)
    #
    #     # Grab server response errors
    #     elif hasattr(e, 'code'):
    #         logger.error("HTTP error code: %d" % e.code)
    #         # "El sitio web no funciona correctamente (error http %d)"
    #         platformtools.dialog_ok(config.get_localized_string(20000), config.get_localized_string(30051) % e.code)
    except WebErrorException as e:
        import traceback
        from core import scrapertools

        logger.error(traceback.format_exc())

        patron = 'File "' + os.path.join(config.get_runtime_path(), "channels", "").replace("\\", "\\\\") + '([^.]+)\.py"'
        canal = scrapertools.find_single_match(traceback.format_exc(), patron)

        platformtools.dialog_ok(
            config.get_localized_string(59985) + canal,
            config.get_localized_string(60013) %(e))
    except:
        import traceback
        from core import scrapertools

        logger.error(traceback.format_exc())

        patron = 'File "' + os.path.join(config.get_runtime_path(), "channels", "").replace("\\", "\\\\") + '([^.]+)\.py"'
        canal = scrapertools.find_single_match(traceback.format_exc(), patron)

        try:
            import xbmc
            if config.get_platform(True)['num_version'] < 14:
                log_name = "xbmc.log"
            else:
                log_name = "kodi.log"
            log_message = config.get_localized_string(50004) + xbmc.translatePath("special://logpath") + log_name
        except:
            log_message = ""

        if canal:
            if item.url:
                if platformtools.dialog_yesno(config.get_localized_string(60087) % canal, config.get_localized_string(60014),
                        log_message, nolabel='ok', yeslabel=config.get_localized_string(70739)):
                    run(Item(action="open_browser", url=item.url))
            else:
                platformtools.dialog_ok(config.get_localized_string(60087) % canal,
                                           config.get_localized_string(60014),
                                           log_message)
        else:
            platformtools.dialog_ok(
                config.get_localized_string(60038),
                config.get_localized_string(60015),
                log_message)
Example #9
0
def scrape(item,
           patron='',
           listGroups=[],
           headers="",
           blacklist="",
           data="",
           patron_block="",
           patronNext="",
           action="findvideos",
           url_host="",
           addVideolibrary=True):
    # patron: the patron to use for scraping page, all capturing group must match with listGroups
    # listGroups: a list containing the scraping info obtained by your patron, in order
    # accepted values are: url, title, thumb, quality, year, plot, duration, genre, rating

    # header: values to pass to request header
    # blacklist: titles that you want to exclude(service articles for example)
    # data: if you want to pass data manually, maybe because you need some custom replacement
    # patron_block: patron to get parts of the page (to scrape with patron attribute),
    #               if you need a "block inside another block" you can create a list, please note that all matches
    #               will be packed as string
    # patronNext: patron for scraping next page link
    # action: if you want results perform an action different from "findvideos", useful when scraping film by genres
    # url_host: string to prepend to scrapedurl, useful when url don't contain host
    # example usage:
    #   import support
    #   itemlist = []
    #   patron = 'blablabla'
    #   headers = [['Referer', host]]
    #   blacklist = 'Request a TV serie!'
    #   return support.scrape(item, itemlist, patron, ['thumb', 'quality', 'url', 'title', 'year', 'plot'],
    #                           headers=headers, blacklist=blacklist)

    itemlist = []

    if not data:
        data = httptools.downloadpage(item.url,
                                      headers=headers).data.replace("'", '"')
        data = re.sub('\n|\t', ' ', data)
        # replace all ' with " and eliminate newline, so we don't need to worry about
        log('DATA =', data)

        block = data

        if patron_block:
            if type(patron_block) == str:
                patron_block = [patron_block]

            for n, regex in enumerate(patron_block):
                blocks = scrapertoolsV2.find_multiple_matches(block, regex)
                block = ""
                for b in blocks:
                    block += "\n" + b
                log('BLOCK ', n, '=', block)
    else:
        block = data
    if patron and listGroups:
        matches = scrapertoolsV2.find_multiple_matches(block, patron)
        log('MATCHES =', matches)

        known_keys = [
            'url', 'title', 'thumb', 'quality', 'year', 'plot', 'duration',
            'genere', 'rating'
        ]
        for match in matches:
            if len(listGroups) > len(match):  # to fix a bug
                match = list(match)
                match.extend([''] * (len(listGroups) - len(match)))

            scraped = {}
            for kk in known_keys:
                val = match[listGroups.index(kk)] if kk in listGroups else ''
                if kk == "url":
                    val = url_host + val
                scraped[kk] = val

            title = scrapertoolsV2.decodeHtmlentities(scraped["title"]).strip()
            plot = scrapertoolsV2.htmlclean(
                scrapertoolsV2.decodeHtmlentities(scraped["plot"]))

            if scraped["quality"]:
                longtitle = '[B]' + title + '[/B] [COLOR blue][' + scraped[
                    "quality"] + '][/COLOR]'
            else:
                longtitle = '[B]' + title + '[/B]'

            if item.infoLabels[
                    "title"] or item.fulltitle:  # if title is set, probably this is a list of episodes or video sources
                infolabels = item.infoLabels
            else:
                infolabels = {}
                if scraped["year"]:
                    infolabels['year'] = scraped["year"]
                if scraped["plot"]:
                    infolabels['plot'] = plot
                if scraped["duration"]:
                    matches = scrapertoolsV2.find_multiple_matches(
                        scraped["duration"],
                        r'([0-9])\s*?(?:[hH]|:|\.|,|\\|\/|\||\s)\s*?([0-9]+)')
                    for h, m in matches:
                        scraped["duration"] = int(h) * 60 + int(m)
                    if not matches:
                        scraped["duration"] = scrapertoolsV2.find_single_match(
                            scraped["duration"], r'(\d+)')
                    infolabels['duration'] = int(scraped["duration"]) * 60
                if scraped["genere"]:
                    genres = scrapertoolsV2.find_multiple_matches(
                        scraped["genere"], '[A-Za-z]+')
                    infolabels['genere'] = ", ".join(genres)
                if scraped["rating"]:
                    infolabels['rating'] = scrapertoolsV2.decodeHtmlentities(
                        scraped["rating"])

            if scraped["title"] not in blacklist:
                it = Item(channel=item.channel,
                          action=action,
                          contentType=item.contentType,
                          title=longtitle,
                          fulltitle=title,
                          show=title,
                          quality=scraped["quality"],
                          url=scraped["url"],
                          infoLabels=infolabels,
                          thumbnail=scraped["thumb"])

                for lg in list(set(listGroups).difference(known_keys)):
                    it.__setattr__(lg, match[listGroups.index(lg)])

                itemlist.append(it)

        if (item.contentType == "episode" and (action != "findvideos" and action != "play")) \
                or (item.contentType == "movie" and action != "play"):
            tmdb.set_infoLabels_itemlist(itemlist, seekTmdb=True)
        else:
            for it in itemlist:
                it.infoLabels = item.infoLabels

        if patronNext:
            nextPage(itemlist, item, data, patronNext, 2)

        if addVideolibrary and (item.infoLabels["title"] or item.fulltitle):
            item.fulltitle = item.infoLabels["title"]
            videolibrary(itemlist, item)

    return itemlist