def load_subreddits_file_into_a_listitem(self):
from utils import compose_list_item
from reddit import subreddit_entry_to_listitem
entries=[]
listing=[]
if os.path.exists(self.subreddits_file):
with open(self.subreddits_file, 'r') as fh:
content = fh.read()
fh.close()
spl = content.split('\n')
for i in range(0, len(spl), 1):
if spl[i]:
subreddit = spl[i].strip()
entries.append(subreddit )
entries.sort()
#log( ' entries count ' + str( len( entries) ) )
for subreddit_entry in entries:
liz=subreddit_entry_to_listitem(subreddit_entry)
liz.setProperty('ACTION_manage_subreddits', build_script('manage_subreddits', subreddit_entry,"","" ) )
listing.append(liz)
li_setting=compose_list_item( translation(32029), "History", "icon_search_subreddit.png", "script", build_script("listRecentlyPlayed", '', '') )
listing.append(li_setting)
li_search=compose_list_item( translation(32016), translation(32016), "icon_search_subreddit.png", "script", build_script("search", '', '') )
listing.append(li_search)
li_setting=compose_list_item( translation(32018), "Program", "icon_settings.png", "script", "Addon.OpenSettings(%s)"%addonID )
listing.append(li_setting)
return listing
def load_subreddits_file_into_a_listitem(self):
from utils import parse_subreddit_entry, build_script, compose_list_item, assemble_reddit_filter_string, prettify_reddit_query
entries=[]
listing=[]
if os.path.exists(self.subreddits_file):
with open(self.subreddits_file, 'r') as fh:
content = fh.read()
fh.close()
spl = content.split('\n')
for i in range(0, len(spl), 1):
if spl[i]:
subreddit = spl[i].strip()
entries.append(subreddit )
entries.sort()
for subreddit_entry in entries:
subreddit, alias, shortcut_description=parse_subreddit_entry(subreddit_entry)
reddit_url= assemble_reddit_filter_string("",subreddit, "yes")
liz = compose_list_item( alias, "", "", "script", build_script("listSubReddit",reddit_url,prettify_reddit_query(alias)) )
liz.setProperty('ACTION_manage_subreddits', build_script('manage_subreddits', subreddit_entry,"","" ) )
listing.append(liz)
return listing
def load_subreddits_file_into_a_listitem(self):
from utils import parse_subreddit_entry, build_script, compose_list_item, assemble_reddit_filter_string, prettify_reddit_query
entries=[]
listing=[]
if os.path.exists(self.subreddits_file):
with open(self.subreddits_file, 'r') as fh:
content = fh.read()
fh.close()
spl = content.split('\n')
for i in range(0, len(spl), 1):
if spl[i]:
subreddit = spl[i].strip()
entries.append(subreddit )
entries.sort()
#log( ' entries count ' + str( len( entries) ) )
for subreddit_entry in entries:
#strip out the alias identifier from the subreddit string retrieved from the file so we can process it.
subreddit, alias, shortcut_description=parse_subreddit_entry(subreddit_entry)
#log( subreddit + " " + shortcut_description )
reddit_url= assemble_reddit_filter_string("",subreddit, "yes")
liz = compose_list_item( alias, "", "", "script", build_script("listSubReddit",reddit_url,prettify_reddit_query(alias)) )
liz.setProperty('ACTION_manage_subreddits', build_script('manage_subreddits', subreddit_entry,"","" ) )
listing.append(liz)
return listing
def load_subreddits_file_into_a_listitem(self):
from utils import compose_list_item, prettify_reddit_query
from reddit import parse_subreddit_entry, assemble_reddit_filter_string
entries = []
listing = []
if os.path.exists(self.subreddits_file):
with open(self.subreddits_file, 'r') as fh:
content = fh.read()
fh.close()
spl = content.split('\n')
for i in range(0, len(spl), 1):
if spl[i]:
subreddit = spl[i].strip()
entries.append(subreddit)
entries.sort()
#log( ' entries count ' + str( len( entries) ) )
for subreddit_entry in entries:
#strip out the alias identifier from the subreddit string retrieved from the file so we can process it.
subreddit, alias, shortcut_description = parse_subreddit_entry(
subreddit_entry)
#log( subreddit + " " + shortcut_description )
reddit_url = assemble_reddit_filter_string("", subreddit, "yes")
liz = compose_list_item(
alias, "", "", "script",
build_script("listSubReddit", reddit_url,
prettify_reddit_query(alias)))
liz.setProperty(
'ACTION_manage_subreddits',
build_script('manage_subreddits', subreddit_entry, "", ""))
listing.append(liz)
return listing
def subreddit_entry_to_listitem(subreddit_entry):
from utils import compose_list_item, build_script, xstr, prettify_reddit_query
addtl_subr_info = {}
nsfw = False
icon = banner = header = public_description = display_name = override_header_image = None
header_ar = 0
addtl_subr_info = ret_sub_info(subreddit_entry)
#strip out the alias identifier from the subreddit string retrieved from the file so we can process it.
entry_type, subreddit, alias, shortcut_description = parse_subreddit_entry(
subreddit_entry)
icon = default_icon = ret_settings_type_default_icon(entry_type)
pretty_label = prettify_reddit_query(alias)
pretty_label = pretty_label.replace('+', ' + ')
if addtl_subr_info:
icon = addtl_subr_info.get('icon_img')
banner = addtl_subr_info.get('banner_img') #rectangular shape
header = addtl_subr_info.get(
'header_img') #square shape from bannerTvImageUrl
header_ar = img_ar(addtl_subr_info.get('header_size'))
if (header_ar > 8) and (
not icon
): #some header_img are very wide. this is to check and override the icon display in the gui
override_header_image = header
banner = header
public_description = xstr(addtl_subr_info.get('public_description',
''))
display_name = xstr(addtl_subr_info.get('display_name', ''))
#log('{} icon={} header={} banner={}'.format( subreddit, repr(icon), repr(header), repr(banner) ))
if entry_type == 'link': #<-- added new ability to have youtube channels as a shortcut on the main screen
#here, the subreddit variable contains a url. we made sure that it points to a youtube channel(ContextMenus.py). that way, there is no need to specify 'channel' when calling listRelatedVideo
liz = compose_list_item(
pretty_label, entry_type, "", "script",
build_script("listRelatedVideo", subreddit, alias))
else: #domain, subreddit, combined, search, multireddit
reddit_url = assemble_reddit_filter_string("", subreddit, "yes")
if entry_type == 'domain':
#remove the identifier that this setting is a domain
pretty_label = re.findall(r'(?::|\/domain\/)(.+)', subreddit)[0]
if subreddit.lower() in ["all", "popular"]:
liz = compose_list_item(
pretty_label, entry_type, "", "script",
build_script("listSubReddit", reddit_url, alias))
else:
if addtl_subr_info: #if we have additional info about this subreddit
#log(repr(addtl_subr_info))
#title=addtl_subr_info.get('title','')+'\n'
#display_name=xstr(addtl_subr_info.get('display_name',''))
#if samealphabetic( title, display_name): title=''
#header_title=xstr(addtl_subr_info.get('header_title',''))
#in reddit_viewer, title, header_title and public_description is shown as plot
nsfw = addtl_subr_info.get('over18')
icon = next(
(item for item in [icon, banner, header] if item),
'') or default_icon #picks the first item that is not None
#icon=icon or default_icon
#log( pretty_label + ' icon=' + icon + ' nsfw='+repr(nsfw))
liz = compose_list_item(
pretty_label, entry_type, "", "script",
build_script("listSubReddit", reddit_url, alias))
else:
liz = compose_list_item(
pretty_label, entry_type, "", "script",
build_script("listSubReddit", reddit_url, alias))
liz.setArt({
"thumb": icon,
"banner": banner,
"fanart": override_header_image
})
liz.setInfo('video', {"Title": display_name, "plot": public_description})
if nsfw:
liz.setProperty('nsfw', 'true')
return liz
def listSubReddit(url, subreddit_key, type_):
from guis import progressBG
from utils import post_is_filtered_out, build_script, compose_list_item, xbmc_notify,prettify_reddit_query, set_query_field
from reddit import reddit_request, has_multiple, assemble_reddit_filter_string, subreddit_icoheader_banner
global GCXM_hasmultiplesubreddit, GCXM_actual_url_used_to_generate_these_posts,GCXM_reddit_query_of_this_gui,GCXM_hasmultipledomain,GCXM_hasmultipleauthor
#the +'s got removed by url conversion
title_bar_name=subreddit_key.replace(' ','+')
if title_bar_name.startswith('?'):
title_bar_name=prettify_reddit_query(title_bar_name)
#log(" title_bar_name %s " %(title_bar_name) )
log("listSubReddit r/%s\n %s" %(title_bar_name,url) )
currentUrl = url
icon=banner=header=None
xbmc_busy()
loading_indicator=progressBG('Loading...')
loading_indicator.update(0,'Retrieving '+subreddit_key)
content = reddit_request(url)
loading_indicator.update(10,subreddit_key )
if not content:
xbmc_busy(False)
loading_indicator.end() #it is important to close xbmcgui.DialogProgressBG
return
threads = []
q_liz = Queue() #output queue (listitem)
content = json.loads(content)
#log("query returned %d items " % len(content['data']['children']) )
posts_count=len(content['data']['children'])
filtered_out_posts=0
hms=has_multiple('subreddit', content['data']['children'])
if hms==False: #r/random and r/randnsfw returns a random subreddit. we need to use the name of this subreddit for the "next page" link.
try: g=content['data']['children'][0]['data']['subreddit']
except ValueError: g=""
except IndexError:
xbmc_busy(False)
loading_indicator.end() #it is important to close xbmcgui.DialogProgressBG
xbmc_notify("List Subreddit",translation(32022))
return
if g:
title_bar_name=g
#preserve the &after string so that functions like play slideshow and play all videos can 'play' the correct page
# extract the &after string from currentUrl -OR- send it with the 'type' argument when calling this function.
currentUrl=assemble_reddit_filter_string('',g) + '&after=' + type_
#put subreddit icon/header in the GUI
icon,banner,header=subreddit_icoheader_banner(g)
GCXM_hasmultiplesubreddit=hms
GCXM_hasmultipledomain=has_multiple('domain', content['data']['children'])
GCXM_hasmultipleauthor=has_multiple('author', content['data']['children'])
GCXM_actual_url_used_to_generate_these_posts=url
GCXM_reddit_query_of_this_gui=currentUrl
for idx, entry in enumerate(content['data']['children']):
try:
if post_is_filtered_out( entry.get('data') ):
filtered_out_posts+=1
continue
domain,domain_count=count_links_from_same_domain( entry ) #count how many same domains we're hitting
delay=compute_anti_dos_delay(domain,domain_count)
#have threads process each reddit post
t = threading.Thread(target=reddit_post_worker, args=(idx, entry,q_liz,delay), name='#t%.2d'%idx)
threads.append(t)
t.start()
except Exception as e:
log(" EXCEPTION:="+ str( sys.exc_info()[0]) + " " + str(e) )
log( repr(domains_d) )
#check the queue to determine progress
break_counter=0 #to avoid infinite loop
expected_listitems=(posts_count-filtered_out_posts)
if expected_listitems>0:
loading_indicator.set_tick_total(expected_listitems)
last_queue_size=0
while q_liz.qsize() < expected_listitems:
if break_counter>=500:
#log('break counter reached limit')
break
#each change in the queue size gets a tick on our progress track
if last_queue_size < q_liz.qsize():
items_added=q_liz.qsize()-last_queue_size
loading_indicator.tick(items_added)
else:
break_counter+=1
last_queue_size=q_liz.qsize()
xbmc.sleep(100)
#wait for all threads to finish before collecting the list items
for idx, t in enumerate(threads):
#log(' joining %s' %t.getName())
t.join(timeout=20) #<-- does not seem to work
xbmc_busy(False)
#compare the number of entries to the returned results
#log( "queue:%d entries:%d" %( q_liz.qsize() , len(content['data']['children'] ) ) )
if q_liz.qsize() != expected_listitems:
#some post might be filtered out.
log('some threads did not return a listitem')
#for t in threads: log('isAlive %s %s' %(t.getName(), repr(t.isAlive()) ) )
#liu=[ qi for qi in sorted(q_liz.queue) ]
li=[ liz for idx,liz in sorted(q_liz.queue) ]
#empty the queue.
with q_liz.mutex:
q_liz.queue.clear()
loading_indicator.end() #it is important to close xbmcgui.DialogProgressBG
try:
#this part makes sure that you load the next page instead of just the first
after=content['data']['after']
o = urlparse.urlparse(currentUrl)
current_url_query = urlparse.parse_qs(o.query)
count=current_url_query.get('count')
if current_url_query.get('count')==None:
#firsttime it is none
count=itemsPerPage
else:
#nexttimes it will be kept incremented with itemsPerPage
try: count=int(current_url_query.get('count')[0]) + int(itemsPerPage)
except ValueError: count=itemsPerPage
nextUrl=set_query_field(currentUrl,'count', count, True)
#log('$$$ nextUrl: ' +nextUrl)
nextUrl=set_query_field(nextUrl, field='after', value=after, replace=True) #(url, field, value, replace=False):
#log('$$$currenturl: ' +currentUrl)
#log('$$$ nextUrl: ' +nextUrl)
liz = compose_list_item( translation(32024), "", "DefaultFolderNextSquare.png", "script", build_script("listSubReddit",nextUrl,title_bar_name,after) )
#for items at the bottom left corner
liz.setArt({ "clearart": "DefaultFolderNextSquare.png" })
liz.setInfo(type='video', infoLabels={"Studio":translation(32024)})
liz.setProperty('link_url', nextUrl )
li.append(liz)
except Exception as e:
log(" EXCEPTzION:="+ str( sys.exc_info()[0]) + " " + str(e) )
xbmc_busy(False)
title_bar_name=urllib.unquote_plus(title_bar_name)
ui=skin_launcher('listSubReddit',
title_bar_name=title_bar_name,
listing=li,
subreddits_file=subredditsFile,
currentUrl=currentUrl,
icon=icon,
banner=banner,
header=header)
ui.doModal()
del ui
def subreddit_entry_to_listitem(subreddit_entry):
from utils import compose_list_item, build_script, xstr, prettify_reddit_query
addtl_subr_info={}
nsfw=False
icon=banner=header=public_description=display_name=override_header_image=None
header_ar=0
addtl_subr_info=ret_sub_info(subreddit_entry)
#strip out the alias identifier from the subreddit string retrieved from the file so we can process it.
entry_type, subreddit, alias, shortcut_description=parse_subreddit_entry(subreddit_entry)
icon=default_icon=ret_settings_type_default_icon(entry_type)
pretty_label=prettify_reddit_query(alias)
pretty_label=pretty_label.replace('+',' + ')
if addtl_subr_info:
icon=addtl_subr_info.get('icon_img')
banner=addtl_subr_info.get('banner_img') #rectangular shape
header=addtl_subr_info.get('header_img') #square shape from bannerTvImageUrl
header_ar=img_ar(addtl_subr_info.get('header_size'))
if (header_ar > 8) and (not icon): #some header_img are very wide. this is to check and override the icon display in the gui
override_header_image=header
banner=header
public_description=xstr( addtl_subr_info.get('public_description',''))
display_name=xstr(addtl_subr_info.get('display_name',''))
#log('{} icon={} header={} banner={}'.format( subreddit, repr(icon), repr(header), repr(banner) ))
if entry_type=='link': #<-- added new ability to have youtube channels as a shortcut on the main screen
#here, the subreddit variable contains a url. we made sure that it points to a youtube channel(ContextMenus.py). that way, there is no need to specify 'channel' when calling listRelatedVideo
liz = compose_list_item( pretty_label, entry_type, "", "script", build_script("listRelatedVideo",subreddit,alias) )
else: #domain, subreddit, combined, search, multireddit
reddit_url=assemble_reddit_filter_string("",subreddit, "yes")
if entry_type=='domain':
#remove the identifier that this setting is a domain
pretty_label=re.findall(r'(?::|\/domain\/)(.+)',subreddit)[0]
if subreddit.lower() in ["all","popular"]:
liz = compose_list_item( pretty_label, entry_type, "", "script", build_script("listSubReddit",reddit_url,alias) )
else:
if addtl_subr_info: #if we have additional info about this subreddit
#log(repr(addtl_subr_info))
#title=addtl_subr_info.get('title','')+'\n'
#display_name=xstr(addtl_subr_info.get('display_name',''))
#if samealphabetic( title, display_name): title=''
#header_title=xstr(addtl_subr_info.get('header_title',''))
#in reddit_viewer, title, header_title and public_description is shown as plot
nsfw=addtl_subr_info.get('over18')
icon=next((item for item in [icon,banner,header] if item ), '') or default_icon #picks the first item that is not None
#icon=icon or default_icon
#log( pretty_label + ' icon=' + icon + ' nsfw='+repr(nsfw))
liz = compose_list_item( pretty_label, entry_type, "", "script", build_script("listSubReddit",reddit_url,alias) )
else:
liz = compose_list_item( pretty_label, entry_type, "", "script", build_script("listSubReddit",reddit_url,alias) )
liz.setArt({ "thumb": icon, "banner":banner, "fanart":override_header_image })
liz.setInfo('video', {"Title":display_name, "plot":public_description} )
if nsfw:
liz.setProperty('nsfw', 'true' )
return liz
def listSubReddit(url, subreddit_key, type_):
from guis import progressBG
from utils import post_is_filtered_out, build_script, compose_list_item, xbmc_notify
from reddit import reddit_request, has_multiple, assemble_reddit_filter_string
global GCXM_hasmultiplesubreddit, GCXM_actual_url_used_to_generate_these_posts, GCXM_reddit_query_of_this_gui, GCXM_hasmultipledomain, GCXM_hasmultipleauthor
#the +'s got removed by url conversion
title_bar_name = subreddit_key.replace(' ', '+')
#log(" title_bar_name %s " %(title_bar_name) )
log("listSubReddit r/%s\n %s" % (title_bar_name, url))
currentUrl = url
icon = banner = header = None
xbmc_busy()
loading_indicator = progressBG('Loading...')
loading_indicator.update(0, 'Retrieving ' + subreddit_key)
content = reddit_request(url)
loading_indicator.update(10, subreddit_key)
if not content:
xbmc_busy(False)
loading_indicator.end(
) #it is important to close xbmcgui.DialogProgressBG
return
threads = []
q_liz = Queue() #output queue (listitem)
content = json.loads(content)
#log("query returned %d items " % len(content['data']['children']) )
posts_count = len(content['data']['children'])
filtered_out_posts = 0
hms = has_multiple('subreddit', content['data']['children'])
if hms == False: #r/random and r/randnsfw returns a random subreddit. we need to use the name of this subreddit for the "next page" link.
try:
g = content['data']['children'][0]['data']['subreddit']
except ValueError:
g = ""
except IndexError:
xbmc_busy(False)
loading_indicator.end(
) #it is important to close xbmcgui.DialogProgressBG
xbmc_notify("List Subreddit", translation(32022))
return
if g:
title_bar_name = g
#preserve the &after string so that functions like play slideshow and play all videos can 'play' the correct page
# extract the &after string from currentUrl -OR- send it with the 'type' argument when calling this function.
currentUrl = assemble_reddit_filter_string('',
g) + '&after=' + type_
#put subreddit icon/header in the GUI
icon, banner, header = subreddit_icoheader_banner(g)
GCXM_hasmultiplesubreddit = hms
GCXM_hasmultipledomain = has_multiple('domain',
content['data']['children'])
GCXM_hasmultipleauthor = has_multiple('author',
content['data']['children'])
GCXM_actual_url_used_to_generate_these_posts = url
GCXM_reddit_query_of_this_gui = currentUrl
for idx, entry in enumerate(content['data']['children']):
try:
#if entry.get('kind')!='t3':
# filtered_out_posts+=1
# continue
if post_is_filtered_out(entry.get('data')):
filtered_out_posts += 1
continue
#have threads process each reddit post
t = threading.Thread(target=reddit_post_worker,
args=(idx, entry, q_liz),
name='#t%.2d' % idx)
threads.append(t)
t.start()
except Exception as e:
log(" EXCEPTION:=" + str(sys.exc_info()[0]) + " " + str(e))
#check the queue to determine progress
break_counter = 0 #to avoid infinite loop
expected_listitems = (posts_count - filtered_out_posts)
if expected_listitems > 0:
loading_indicator.set_tick_total(expected_listitems)
last_queue_size = 0
while q_liz.qsize() < expected_listitems:
if break_counter >= 100:
break
#each change in the queue size gets a tick on our progress track
if last_queue_size < q_liz.qsize():
items_added = q_liz.qsize() - last_queue_size
loading_indicator.tick(items_added)
else:
break_counter += 1
last_queue_size = q_liz.qsize()
xbmc.sleep(100)
#wait for all threads to finish before collecting the list items
for idx, t in enumerate(threads):
#log(' joining %s' %t.getName())
t.join(timeout=20)
xbmc_busy(False)
#compare the number of entries to the returned results
#log( "queue:%d entries:%d" %( q_liz.qsize() , len(content['data']['children'] ) ) )
if q_liz.qsize() != expected_listitems:
#some post might be filtered out.
log('some threads did not return a listitem')
#for t in threads: log('isAlive %s %s' %(t.getName(), repr(t.isAlive()) ) )
#liu=[ qi for qi in sorted(q_liz.queue) ]
li = [liz for idx, liz in sorted(q_liz.queue)]
#empty the queue.
with q_liz.mutex:
q_liz.queue.clear()
loading_indicator.end() #it is important to close xbmcgui.DialogProgressBG
try:
#this part makes sure that you load the next page instead of just the first
after = ""
after = content['data']['after']
if after:
if "&after=" in currentUrl:
nextUrl = currentUrl[:currentUrl.find("&after="
)] + "&after=" + after
else:
nextUrl = currentUrl + "&after=" + after
liz = compose_list_item(
translation(32004), "", "DefaultFolderNextSquare.png",
"script",
build_script("listSubReddit", nextUrl, title_bar_name, after))
#for items at the bottom left corner
liz.setArt({"clearart": "DefaultFolderNextSquare.png"})
liz.setInfo(type='video',
infoLabels={"Studio": translation(32004)})
liz.setProperty('link_url', nextUrl)
li.append(liz)
except Exception as e:
log(" EXCEPTzION:=" + str(sys.exc_info()[0]) + " " + str(e))
xbmc_busy(False)
title_bar_name = urllib.unquote_plus(title_bar_name)
ui = skin_launcher('listSubReddit',
title_bar_name=title_bar_name,
listing=li,
subreddits_file=subredditsFile,
currentUrl=currentUrl,
icon=icon,
banner=banner,
header=header)
ui.doModal()
del ui