def load_subreddits_file_into_a_listitem(self): from utils import compose_list_item from reddit import subreddit_entry_to_listitem entries=[] listing=[] if os.path.exists(self.subreddits_file): with open(self.subreddits_file, 'r') as fh: content = fh.read() fh.close() spl = content.split('\n') for i in range(0, len(spl), 1): if spl[i]: subreddit = spl[i].strip() entries.append(subreddit ) entries.sort() #log( ' entries count ' + str( len( entries) ) ) for subreddit_entry in entries: liz=subreddit_entry_to_listitem(subreddit_entry) liz.setProperty('ACTION_manage_subreddits', build_script('manage_subreddits', subreddit_entry,"","" ) ) listing.append(liz) li_setting=compose_list_item( translation(32029), "History", "icon_search_subreddit.png", "script", build_script("listRecentlyPlayed", '', '') ) listing.append(li_setting) li_search=compose_list_item( translation(32016), translation(32016), "icon_search_subreddit.png", "script", build_script("search", '', '') ) listing.append(li_search) li_setting=compose_list_item( translation(32018), "Program", "icon_settings.png", "script", "Addon.OpenSettings(%s)"%addonID ) listing.append(li_setting) return listing
def load_subreddits_file_into_a_listitem(self): from utils import parse_subreddit_entry, build_script, compose_list_item, assemble_reddit_filter_string, prettify_reddit_query entries=[] listing=[] if os.path.exists(self.subreddits_file): with open(self.subreddits_file, 'r') as fh: content = fh.read() fh.close() spl = content.split('\n') for i in range(0, len(spl), 1): if spl[i]: subreddit = spl[i].strip() entries.append(subreddit ) entries.sort() for subreddit_entry in entries: subreddit, alias, shortcut_description=parse_subreddit_entry(subreddit_entry) reddit_url= assemble_reddit_filter_string("",subreddit, "yes") liz = compose_list_item( alias, "", "", "script", build_script("listSubReddit",reddit_url,prettify_reddit_query(alias)) ) liz.setProperty('ACTION_manage_subreddits', build_script('manage_subreddits', subreddit_entry,"","" ) ) listing.append(liz) return listing
def load_subreddits_file_into_a_listitem(self): from utils import parse_subreddit_entry, build_script, compose_list_item, assemble_reddit_filter_string, prettify_reddit_query entries=[] listing=[] if os.path.exists(self.subreddits_file): with open(self.subreddits_file, 'r') as fh: content = fh.read() fh.close() spl = content.split('\n') for i in range(0, len(spl), 1): if spl[i]: subreddit = spl[i].strip() entries.append(subreddit ) entries.sort() #log( ' entries count ' + str( len( entries) ) ) for subreddit_entry in entries: #strip out the alias identifier from the subreddit string retrieved from the file so we can process it. subreddit, alias, shortcut_description=parse_subreddit_entry(subreddit_entry) #log( subreddit + " " + shortcut_description ) reddit_url= assemble_reddit_filter_string("",subreddit, "yes") liz = compose_list_item( alias, "", "", "script", build_script("listSubReddit",reddit_url,prettify_reddit_query(alias)) ) liz.setProperty('ACTION_manage_subreddits', build_script('manage_subreddits', subreddit_entry,"","" ) ) listing.append(liz) return listing
def load_subreddits_file_into_a_listitem(self): from utils import compose_list_item, prettify_reddit_query from reddit import parse_subreddit_entry, assemble_reddit_filter_string entries = [] listing = [] if os.path.exists(self.subreddits_file): with open(self.subreddits_file, 'r') as fh: content = fh.read() fh.close() spl = content.split('\n') for i in range(0, len(spl), 1): if spl[i]: subreddit = spl[i].strip() entries.append(subreddit) entries.sort() #log( ' entries count ' + str( len( entries) ) ) for subreddit_entry in entries: #strip out the alias identifier from the subreddit string retrieved from the file so we can process it. subreddit, alias, shortcut_description = parse_subreddit_entry( subreddit_entry) #log( subreddit + " " + shortcut_description ) reddit_url = assemble_reddit_filter_string("", subreddit, "yes") liz = compose_list_item( alias, "", "", "script", build_script("listSubReddit", reddit_url, prettify_reddit_query(alias))) liz.setProperty( 'ACTION_manage_subreddits', build_script('manage_subreddits', subreddit_entry, "", "")) listing.append(liz) return listing
def subreddit_entry_to_listitem(subreddit_entry): from utils import compose_list_item, build_script, xstr, prettify_reddit_query addtl_subr_info = {} nsfw = False icon = banner = header = public_description = display_name = override_header_image = None header_ar = 0 addtl_subr_info = ret_sub_info(subreddit_entry) #strip out the alias identifier from the subreddit string retrieved from the file so we can process it. entry_type, subreddit, alias, shortcut_description = parse_subreddit_entry( subreddit_entry) icon = default_icon = ret_settings_type_default_icon(entry_type) pretty_label = prettify_reddit_query(alias) pretty_label = pretty_label.replace('+', ' + ') if addtl_subr_info: icon = addtl_subr_info.get('icon_img') banner = addtl_subr_info.get('banner_img') #rectangular shape header = addtl_subr_info.get( 'header_img') #square shape from bannerTvImageUrl header_ar = img_ar(addtl_subr_info.get('header_size')) if (header_ar > 8) and ( not icon ): #some header_img are very wide. this is to check and override the icon display in the gui override_header_image = header banner = header public_description = xstr(addtl_subr_info.get('public_description', '')) display_name = xstr(addtl_subr_info.get('display_name', '')) #log('{} icon={} header={} banner={}'.format( subreddit, repr(icon), repr(header), repr(banner) )) if entry_type == 'link': #<-- added new ability to have youtube channels as a shortcut on the main screen #here, the subreddit variable contains a url. we made sure that it points to a youtube channel(ContextMenus.py). that way, there is no need to specify 'channel' when calling listRelatedVideo liz = compose_list_item( pretty_label, entry_type, "", "script", build_script("listRelatedVideo", subreddit, alias)) else: #domain, subreddit, combined, search, multireddit reddit_url = assemble_reddit_filter_string("", subreddit, "yes") if entry_type == 'domain': #remove the identifier that this setting is a domain pretty_label = re.findall(r'(?::|\/domain\/)(.+)', subreddit)[0] if subreddit.lower() in ["all", "popular"]: liz = compose_list_item( pretty_label, entry_type, "", "script", build_script("listSubReddit", reddit_url, alias)) else: if addtl_subr_info: #if we have additional info about this subreddit #log(repr(addtl_subr_info)) #title=addtl_subr_info.get('title','')+'\n' #display_name=xstr(addtl_subr_info.get('display_name','')) #if samealphabetic( title, display_name): title='' #header_title=xstr(addtl_subr_info.get('header_title','')) #in reddit_viewer, title, header_title and public_description is shown as plot nsfw = addtl_subr_info.get('over18') icon = next( (item for item in [icon, banner, header] if item), '') or default_icon #picks the first item that is not None #icon=icon or default_icon #log( pretty_label + ' icon=' + icon + ' nsfw='+repr(nsfw)) liz = compose_list_item( pretty_label, entry_type, "", "script", build_script("listSubReddit", reddit_url, alias)) else: liz = compose_list_item( pretty_label, entry_type, "", "script", build_script("listSubReddit", reddit_url, alias)) liz.setArt({ "thumb": icon, "banner": banner, "fanart": override_header_image }) liz.setInfo('video', {"Title": display_name, "plot": public_description}) if nsfw: liz.setProperty('nsfw', 'true') return liz
def listSubReddit(url, subreddit_key, type_): from guis import progressBG from utils import post_is_filtered_out, build_script, compose_list_item, xbmc_notify,prettify_reddit_query, set_query_field from reddit import reddit_request, has_multiple, assemble_reddit_filter_string, subreddit_icoheader_banner global GCXM_hasmultiplesubreddit, GCXM_actual_url_used_to_generate_these_posts,GCXM_reddit_query_of_this_gui,GCXM_hasmultipledomain,GCXM_hasmultipleauthor #the +'s got removed by url conversion title_bar_name=subreddit_key.replace(' ','+') if title_bar_name.startswith('?'): title_bar_name=prettify_reddit_query(title_bar_name) #log(" title_bar_name %s " %(title_bar_name) ) log("listSubReddit r/%s\n %s" %(title_bar_name,url) ) currentUrl = url icon=banner=header=None xbmc_busy() loading_indicator=progressBG('Loading...') loading_indicator.update(0,'Retrieving '+subreddit_key) content = reddit_request(url) loading_indicator.update(10,subreddit_key ) if not content: xbmc_busy(False) loading_indicator.end() #it is important to close xbmcgui.DialogProgressBG return threads = [] q_liz = Queue() #output queue (listitem) content = json.loads(content) #log("query returned %d items " % len(content['data']['children']) ) posts_count=len(content['data']['children']) filtered_out_posts=0 hms=has_multiple('subreddit', content['data']['children']) if hms==False: #r/random and r/randnsfw returns a random subreddit. we need to use the name of this subreddit for the "next page" link. try: g=content['data']['children'][0]['data']['subreddit'] except ValueError: g="" except IndexError: xbmc_busy(False) loading_indicator.end() #it is important to close xbmcgui.DialogProgressBG xbmc_notify("List Subreddit",translation(32022)) return if g: title_bar_name=g #preserve the &after string so that functions like play slideshow and play all videos can 'play' the correct page # extract the &after string from currentUrl -OR- send it with the 'type' argument when calling this function. currentUrl=assemble_reddit_filter_string('',g) + '&after=' + type_ #put subreddit icon/header in the GUI icon,banner,header=subreddit_icoheader_banner(g) GCXM_hasmultiplesubreddit=hms GCXM_hasmultipledomain=has_multiple('domain', content['data']['children']) GCXM_hasmultipleauthor=has_multiple('author', content['data']['children']) GCXM_actual_url_used_to_generate_these_posts=url GCXM_reddit_query_of_this_gui=currentUrl for idx, entry in enumerate(content['data']['children']): try: if post_is_filtered_out( entry.get('data') ): filtered_out_posts+=1 continue domain,domain_count=count_links_from_same_domain( entry ) #count how many same domains we're hitting delay=compute_anti_dos_delay(domain,domain_count) #have threads process each reddit post t = threading.Thread(target=reddit_post_worker, args=(idx, entry,q_liz,delay), name='#t%.2d'%idx) threads.append(t) t.start() except Exception as e: log(" EXCEPTION:="+ str( sys.exc_info()[0]) + " " + str(e) ) log( repr(domains_d) ) #check the queue to determine progress break_counter=0 #to avoid infinite loop expected_listitems=(posts_count-filtered_out_posts) if expected_listitems>0: loading_indicator.set_tick_total(expected_listitems) last_queue_size=0 while q_liz.qsize() < expected_listitems: if break_counter>=500: #log('break counter reached limit') break #each change in the queue size gets a tick on our progress track if last_queue_size < q_liz.qsize(): items_added=q_liz.qsize()-last_queue_size loading_indicator.tick(items_added) else: break_counter+=1 last_queue_size=q_liz.qsize() xbmc.sleep(100) #wait for all threads to finish before collecting the list items for idx, t in enumerate(threads): #log(' joining %s' %t.getName()) t.join(timeout=20) #<-- does not seem to work xbmc_busy(False) #compare the number of entries to the returned results #log( "queue:%d entries:%d" %( q_liz.qsize() , len(content['data']['children'] ) ) ) if q_liz.qsize() != expected_listitems: #some post might be filtered out. log('some threads did not return a listitem') #for t in threads: log('isAlive %s %s' %(t.getName(), repr(t.isAlive()) ) ) #liu=[ qi for qi in sorted(q_liz.queue) ] li=[ liz for idx,liz in sorted(q_liz.queue) ] #empty the queue. with q_liz.mutex: q_liz.queue.clear() loading_indicator.end() #it is important to close xbmcgui.DialogProgressBG try: #this part makes sure that you load the next page instead of just the first after=content['data']['after'] o = urlparse.urlparse(currentUrl) current_url_query = urlparse.parse_qs(o.query) count=current_url_query.get('count') if current_url_query.get('count')==None: #firsttime it is none count=itemsPerPage else: #nexttimes it will be kept incremented with itemsPerPage try: count=int(current_url_query.get('count')[0]) + int(itemsPerPage) except ValueError: count=itemsPerPage nextUrl=set_query_field(currentUrl,'count', count, True) #log('$$$ nextUrl: ' +nextUrl) nextUrl=set_query_field(nextUrl, field='after', value=after, replace=True) #(url, field, value, replace=False): #log('$$$currenturl: ' +currentUrl) #log('$$$ nextUrl: ' +nextUrl) liz = compose_list_item( translation(32024), "", "DefaultFolderNextSquare.png", "script", build_script("listSubReddit",nextUrl,title_bar_name,after) ) #for items at the bottom left corner liz.setArt({ "clearart": "DefaultFolderNextSquare.png" }) liz.setInfo(type='video', infoLabels={"Studio":translation(32024)}) liz.setProperty('link_url', nextUrl ) li.append(liz) except Exception as e: log(" EXCEPTzION:="+ str( sys.exc_info()[0]) + " " + str(e) ) xbmc_busy(False) title_bar_name=urllib.unquote_plus(title_bar_name) ui=skin_launcher('listSubReddit', title_bar_name=title_bar_name, listing=li, subreddits_file=subredditsFile, currentUrl=currentUrl, icon=icon, banner=banner, header=header) ui.doModal() del ui
def subreddit_entry_to_listitem(subreddit_entry): from utils import compose_list_item, build_script, xstr, prettify_reddit_query addtl_subr_info={} nsfw=False icon=banner=header=public_description=display_name=override_header_image=None header_ar=0 addtl_subr_info=ret_sub_info(subreddit_entry) #strip out the alias identifier from the subreddit string retrieved from the file so we can process it. entry_type, subreddit, alias, shortcut_description=parse_subreddit_entry(subreddit_entry) icon=default_icon=ret_settings_type_default_icon(entry_type) pretty_label=prettify_reddit_query(alias) pretty_label=pretty_label.replace('+',' + ') if addtl_subr_info: icon=addtl_subr_info.get('icon_img') banner=addtl_subr_info.get('banner_img') #rectangular shape header=addtl_subr_info.get('header_img') #square shape from bannerTvImageUrl header_ar=img_ar(addtl_subr_info.get('header_size')) if (header_ar > 8) and (not icon): #some header_img are very wide. this is to check and override the icon display in the gui override_header_image=header banner=header public_description=xstr( addtl_subr_info.get('public_description','')) display_name=xstr(addtl_subr_info.get('display_name','')) #log('{} icon={} header={} banner={}'.format( subreddit, repr(icon), repr(header), repr(banner) )) if entry_type=='link': #<-- added new ability to have youtube channels as a shortcut on the main screen #here, the subreddit variable contains a url. we made sure that it points to a youtube channel(ContextMenus.py). that way, there is no need to specify 'channel' when calling listRelatedVideo liz = compose_list_item( pretty_label, entry_type, "", "script", build_script("listRelatedVideo",subreddit,alias) ) else: #domain, subreddit, combined, search, multireddit reddit_url=assemble_reddit_filter_string("",subreddit, "yes") if entry_type=='domain': #remove the identifier that this setting is a domain pretty_label=re.findall(r'(?::|\/domain\/)(.+)',subreddit)[0] if subreddit.lower() in ["all","popular"]: liz = compose_list_item( pretty_label, entry_type, "", "script", build_script("listSubReddit",reddit_url,alias) ) else: if addtl_subr_info: #if we have additional info about this subreddit #log(repr(addtl_subr_info)) #title=addtl_subr_info.get('title','')+'\n' #display_name=xstr(addtl_subr_info.get('display_name','')) #if samealphabetic( title, display_name): title='' #header_title=xstr(addtl_subr_info.get('header_title','')) #in reddit_viewer, title, header_title and public_description is shown as plot nsfw=addtl_subr_info.get('over18') icon=next((item for item in [icon,banner,header] if item ), '') or default_icon #picks the first item that is not None #icon=icon or default_icon #log( pretty_label + ' icon=' + icon + ' nsfw='+repr(nsfw)) liz = compose_list_item( pretty_label, entry_type, "", "script", build_script("listSubReddit",reddit_url,alias) ) else: liz = compose_list_item( pretty_label, entry_type, "", "script", build_script("listSubReddit",reddit_url,alias) ) liz.setArt({ "thumb": icon, "banner":banner, "fanart":override_header_image }) liz.setInfo('video', {"Title":display_name, "plot":public_description} ) if nsfw: liz.setProperty('nsfw', 'true' ) return liz
def listSubReddit(url, subreddit_key, type_): from guis import progressBG from utils import post_is_filtered_out, build_script, compose_list_item, xbmc_notify from reddit import reddit_request, has_multiple, assemble_reddit_filter_string global GCXM_hasmultiplesubreddit, GCXM_actual_url_used_to_generate_these_posts, GCXM_reddit_query_of_this_gui, GCXM_hasmultipledomain, GCXM_hasmultipleauthor #the +'s got removed by url conversion title_bar_name = subreddit_key.replace(' ', '+') #log(" title_bar_name %s " %(title_bar_name) ) log("listSubReddit r/%s\n %s" % (title_bar_name, url)) currentUrl = url icon = banner = header = None xbmc_busy() loading_indicator = progressBG('Loading...') loading_indicator.update(0, 'Retrieving ' + subreddit_key) content = reddit_request(url) loading_indicator.update(10, subreddit_key) if not content: xbmc_busy(False) loading_indicator.end( ) #it is important to close xbmcgui.DialogProgressBG return threads = [] q_liz = Queue() #output queue (listitem) content = json.loads(content) #log("query returned %d items " % len(content['data']['children']) ) posts_count = len(content['data']['children']) filtered_out_posts = 0 hms = has_multiple('subreddit', content['data']['children']) if hms == False: #r/random and r/randnsfw returns a random subreddit. we need to use the name of this subreddit for the "next page" link. try: g = content['data']['children'][0]['data']['subreddit'] except ValueError: g = "" except IndexError: xbmc_busy(False) loading_indicator.end( ) #it is important to close xbmcgui.DialogProgressBG xbmc_notify("List Subreddit", translation(32022)) return if g: title_bar_name = g #preserve the &after string so that functions like play slideshow and play all videos can 'play' the correct page # extract the &after string from currentUrl -OR- send it with the 'type' argument when calling this function. currentUrl = assemble_reddit_filter_string('', g) + '&after=' + type_ #put subreddit icon/header in the GUI icon, banner, header = subreddit_icoheader_banner(g) GCXM_hasmultiplesubreddit = hms GCXM_hasmultipledomain = has_multiple('domain', content['data']['children']) GCXM_hasmultipleauthor = has_multiple('author', content['data']['children']) GCXM_actual_url_used_to_generate_these_posts = url GCXM_reddit_query_of_this_gui = currentUrl for idx, entry in enumerate(content['data']['children']): try: #if entry.get('kind')!='t3': # filtered_out_posts+=1 # continue if post_is_filtered_out(entry.get('data')): filtered_out_posts += 1 continue #have threads process each reddit post t = threading.Thread(target=reddit_post_worker, args=(idx, entry, q_liz), name='#t%.2d' % idx) threads.append(t) t.start() except Exception as e: log(" EXCEPTION:=" + str(sys.exc_info()[0]) + " " + str(e)) #check the queue to determine progress break_counter = 0 #to avoid infinite loop expected_listitems = (posts_count - filtered_out_posts) if expected_listitems > 0: loading_indicator.set_tick_total(expected_listitems) last_queue_size = 0 while q_liz.qsize() < expected_listitems: if break_counter >= 100: break #each change in the queue size gets a tick on our progress track if last_queue_size < q_liz.qsize(): items_added = q_liz.qsize() - last_queue_size loading_indicator.tick(items_added) else: break_counter += 1 last_queue_size = q_liz.qsize() xbmc.sleep(100) #wait for all threads to finish before collecting the list items for idx, t in enumerate(threads): #log(' joining %s' %t.getName()) t.join(timeout=20) xbmc_busy(False) #compare the number of entries to the returned results #log( "queue:%d entries:%d" %( q_liz.qsize() , len(content['data']['children'] ) ) ) if q_liz.qsize() != expected_listitems: #some post might be filtered out. log('some threads did not return a listitem') #for t in threads: log('isAlive %s %s' %(t.getName(), repr(t.isAlive()) ) ) #liu=[ qi for qi in sorted(q_liz.queue) ] li = [liz for idx, liz in sorted(q_liz.queue)] #empty the queue. with q_liz.mutex: q_liz.queue.clear() loading_indicator.end() #it is important to close xbmcgui.DialogProgressBG try: #this part makes sure that you load the next page instead of just the first after = "" after = content['data']['after'] if after: if "&after=" in currentUrl: nextUrl = currentUrl[:currentUrl.find("&after=" )] + "&after=" + after else: nextUrl = currentUrl + "&after=" + after liz = compose_list_item( translation(32004), "", "DefaultFolderNextSquare.png", "script", build_script("listSubReddit", nextUrl, title_bar_name, after)) #for items at the bottom left corner liz.setArt({"clearart": "DefaultFolderNextSquare.png"}) liz.setInfo(type='video', infoLabels={"Studio": translation(32004)}) liz.setProperty('link_url', nextUrl) li.append(liz) except Exception as e: log(" EXCEPTzION:=" + str(sys.exc_info()[0]) + " " + str(e)) xbmc_busy(False) title_bar_name = urllib.unquote_plus(title_bar_name) ui = skin_launcher('listSubReddit', title_bar_name=title_bar_name, listing=li, subreddits_file=subredditsFile, currentUrl=currentUrl, icon=icon, banner=banner, header=header) ui.doModal() del ui