def add_root_shows(url, doubles = []): root_dict = {} root_data = _connection.getURL(url) root_tree = BeautifulSoup(root_data, 'html5lib') root_menu = root_tree.find('ol', class_ = 'lst').find_all('a') for root_item in root_menu: try: root_name = root_item.find('div', class_ = 'text-block').span.string except: root_name = root_item.text season = re.compile(r' \(Season \d+\)') root_name = season.sub('', root_name).strip() if 'series.jhtml' in root_item['href']: season_url = root_item['href'].replace('series.jhtml', 'video.jhtml?sort=descend') else: season_url = root_item['href'] + 'video.jhtml?sort=descend' if BASE not in season_url: season_url = BASE + season_url if season_url.split('season')[0].replace('video.jhtml?sort=descend','') not in doubles: tvdb_name = _common.get_show_data(root_name,SITE, 'seasons')[-1] if tvdb_name not in root_dict.keys(): root_dict[tvdb_name] = season_url else: root_dict[tvdb_name] = root_dict[tvdb_name] + ',' + season_url doubles.append(season_url.split('season')[0].replace('video.jhtml?sort=descend','')) for root_name, season_url in root_dict.iteritems(): _common.add_show(root_name, SITE, 'seasons', season_url) next = root_tree.find('a', class_ = 'page-next') if next: doubles = add_root_shows(BASE + next['href'], doubles) return doubles
def masterlist(): master_start = 0 master_count = 200 master_db = [] master_dict = {} master_check = [] master_menu = simplejson.loads(_connection.getURL(SHOWS, header = {'X-Requested-With' : 'XMLHttpRequest'})) for master_item in master_menu.itervalues(): for master_item in master_item: master_check.append(master_item['title']) while master_start < master_count: master_data = cove.programs.filter(fields = 'mediafiles', order_by = 'title', limit_start = master_start) master_menu = master_data['results'] master_count = master_data['count'] master_stop = master_data['stop'] del master_data for master_item2 in master_menu: website = master_item2['website'] if website is None: website = '' if master_item2['title'] in master_check and ('PBS Kids' != master_item2['nola_root']) and ('blog' not in website): master_name = _common.smart_utf8(master_item2['title']) tvdb_name = _common.get_show_data(master_name, SITE, 'seasons')[-1] season_url = re.compile('/cove/v1/programs/(.*?)/').findall(master_item2['resource_uri'])[0] if tvdb_name not in master_dict.keys(): master_dict[tvdb_name] = _common.smart_unicode(master_name) + '#' +season_url else: master_dict[tvdb_name] = master_dict[tvdb_name] + ',' + master_name + '#' + season_url master_start = master_stop for master_name in master_dict: season_url = master_dict[master_name] master_db.append((master_name, SITE, 'seasons', season_url)) return master_db
def add_master_shows(url, doubles = [], master_db = []): master_dict = {} master_data = _connection.getURL(url) master_tree = BeautifulSoup(master_data, 'html5lib') master_menu = master_tree.find('ol', class_ = 'lst').find_all('a') for master_item in master_menu: try: master_name = master_item.find('div', class_ = 'text-block').span.string except: master_name = master_item.text season = re.compile(r' \(Season \d+\)') master_name = season.sub('', master_name).strip() season_url = master_item['href'].replace('series.jhtml', 'video.jhtml?sort=descend') if BASE not in season_url: season_url = BASE + season_url if season_url.split('season')[0].replace('video.jhtml?sort=descend','') not in doubles: tvdb_name = _common.get_show_data(master_name,SITE, 'seasons')[-1] if tvdb_name not in master_dict.keys(): master_dict[tvdb_name] = season_url else: master_dict[tvdb_name] = master_dict[tvdb_name] + ',' + season_url doubles.append(season_url.split('season')[0].replace('video.jhtml?sort=descend','')) for master_name, season_url in master_dict.iteritems(): master_db.append((master_name, SITE, 'seasons', season_url)) next = master_tree.find('a', class_ = 'page-next') if next: master_db, doubles = add_master_shows(BASE + next['href'], doubles, master_db) return master_db, doubles
def masterlist(SITE, SHOWS, SPECIALS = None): master_db = [] root_dict = {} root_url = SHOWS root_data = _connection.getURL(root_url) root_tree = BeautifulSoup(root_data, 'html.parser', parse_only = SoupStrainer('div', id = 'grid-frame')) root_menu = root_tree.find_all('div', class_ = 'media-module') for root_item in root_menu: root_name = root_item.find('div', class_ = 'title').text season_url = BASE + root_item.a['href'] if '-1' not in season_url: tvdb_name = _common.get_show_data(root_name, SITE, 'episodes')[-1] root_name = root_name + '#' + season_url if tvdb_name not in root_dict.keys(): root_dict[tvdb_name] = root_name else: root_dict[tvdb_name] = root_dict[tvdb_name] + ',' + root_name for root_name in root_dict: season_url = root_dict[root_name] master_db.append((root_name, SITE, 'episodes', season_url)) more = root_tree.find('a', class_ = 'load-more') if more: master_db.extend(masterlist(SITE, BASE + more['href'])) return master_db
def masterlist(SITE, SHOWS, SPECIALS=None): master_db = [] root_dict = {} root_url = SHOWS root_data = _connection.getURL(root_url) root_tree = BeautifulSoup(root_data, 'html.parser', parse_only=SoupStrainer('div', id='grid-frame')) root_menu = root_tree.find_all('div', class_='media-module') for root_item in root_menu: root_name = root_item.find('div', class_='title').text season_url = BASE + root_item.a['href'] if '-1' not in season_url: tvdb_name = _common.get_show_data(root_name, SITE, 'episodes')[-1] root_name = root_name + '#' + season_url if tvdb_name not in root_dict.keys(): root_dict[tvdb_name] = root_name else: root_dict[tvdb_name] = root_dict[tvdb_name] + ',' + root_name for root_name in root_dict: season_url = root_dict[root_name] master_db.append((root_name, SITE, 'episodes', season_url)) more = root_tree.find('a', class_='load-more') if more: master_db.extend(masterlist(SITE, BASE + more['href'])) return master_db
def add_master_shows(url, doubles = [], master_db = []): master_dict = {} root_dict = {} for i in range(ord('a') - 1, ord('z') + 1): if i < ord('a'): url = SHOWS else: url = SHOWSAZ % chr(i) master_data = _connection.getURL(url) master_tree = BeautifulSoup(master_data, 'html5lib') master_menu = master_tree.find_all('a', attrs = {'data-report' : 'SHOWS_HUB:SHOWS_AZ:SHOW'}) for master_item in master_menu: master_name = master_item.text season = re.compile(r' \(Season \d+\)') master_name = season.sub('', master_name).strip() season_url = master_item['href'].replace('series.jhtml', 'video.jhtml?sort=descend') if BASE not in season_url: season_url = BASE + season_url if season_url.split('season')[0].replace('video.jhtml?sort=descend','') not in doubles: tvdb_name = _common.get_show_data(master_name,SITE, 'seasons')[-1] if tvdb_name not in master_dict.keys(): master_dict[tvdb_name] = season_url else: master_dict[tvdb_name] = master_dict[tvdb_name] + ',' + season_url doubles.append(season_url.split('season')[0].replace('video.jhtml?sort=descend','')) for master_name, season_url in master_dict.iteritems(): master_db.append((master_name, SITE, 'seasons', season_url)) next = master_tree.find('a', class_ = 'page-next') if next: master_db, doubles = add_master_shows(BASE + next['href'], doubles, master_db) return master_db, doubles
def masterlist(): master_db = [] master_dict = {} master_data = _connection.getURL(SHOWS) master_tree = BeautifulSoup(master_data, 'html.parser') master_menu = master_tree.allcollections.find_all('collection') for master_item in master_menu: master_name = _common.smart_utf8(master_item['name']) if '[AD]' not in master_name: tvdb_name = _common.get_show_data(master_name, SITE, 'seasons')[-1] season_url = master_item['id'] season_url = season_url + '#tveepisodes=' try: for season in master_item.tveepisodes.find_all('season'): season_url = season_url + '-' + season['number'] except: pass season_url = season_url + '#clips=' try: for season in master_item.clips.find_all('season'): if season['number'] != '': season_url = season_url + '-' + season['number'] else: season_url = season_url + '-' + '*' except: pass master_db.append((master_name, SITE, 'seasons', season_url)) return master_db
def masterlist(): master_db = [] master_dict = {} master_data = _connection.getURL(SHOWS) master_tree = plistlib.readPlistFromString(master_data) for master_item in master_tree: master_name = _common.smart_utf8(master_item["name"]) tvdb_name = _common.get_show_data(master_name, SITE, "seasons")[-1] if tvdb_name not in master_dict.keys(): master_dict[tvdb_name] = master_item["show-id"] else: master_dict[tvdb_name] = master_dict[tvdb_name] + "," + master_item["show-id"] for master_name in master_dict: season_url = master_dict[master_name] master_db.append((master_name, SITE, "seasons", season_url)) return master_db
def masterlist(): master_db = [] master_dict = {} master_data = _connection.getURL(SHOWS) master_tree = plistlib.readPlistFromString(master_data) for master_item in master_tree: master_name = _common.smart_utf8(master_item['name']) tvdb_name = _common.get_show_data(master_name, SITE, 'seasons')[-1] if tvdb_name not in master_dict.keys(): master_dict[tvdb_name] = master_item['show-id'] else: master_dict[tvdb_name] = master_dict[tvdb_name] + ',' + master_item['show-id'] for master_name in master_dict: season_url = master_dict[master_name] master_db.append((master_name, SITE, 'seasons', season_url)) return master_db
def rootlist(): root_dict = {} root_data = _connection.getURL(SHOWS) root_tree = BeautifulSoup(root_data, 'html5lib') root_menu = root_tree.allcollections.find_all('collection') for root_item in root_menu: root_name = _common.smart_utf8(root_item['name']) tvdb_name = _common.get_show_data(root_name, SITE, 'seasons')[-1] if tvdb_name not in root_dict.keys(): root_dict[tvdb_name] = root_item['id'] else: root_dict[tvdb_name] = root_dict[tvdb_name] + ',' + root_item['id'] for root_name in root_dict: season_url = root_dict[root_name] _common.add_show(root_name, SITE, 'seasons', season_url) _common.set_view('tvshows')
def masterlist(): master_db = [] master_dict = {} master_data = _connection.getURL(SHOWS) master_tree = BeautifulSoup(master_data, 'html5lib') master_menu = master_tree.allcollections.find_all('collection') for master_item in master_menu: master_name = _common.smart_utf8(master_item['name']) tvdb_name = _common.get_show_data(master_name, SITE, 'seasons')[-1] if tvdb_name not in master_dict.keys(): master_dict[tvdb_name] = master_item['id'] else: master_dict[tvdb_name] = master_dict[tvdb_name] + ',' + master_item['id'] for master_name in master_dict: season_url = master_dict[master_name] master_db.append((master_name, SITE, 'seasons', season_url)) return master_db
def rootlist(): root_dict = {} root_data = _connection.getURL(SHOWS) root_tree = BeautifulSoup(root_data, 'html5lib') root_section = root_tree.find_all('div', class_ = 'primetime_and_originals') for section in root_section: root_menu = section.find_all('a', text = True) for root_item in root_menu: root_name = root_item.text tvdb_name = _common.get_show_data(root_name,SITE, 'seasons')[-1] season_url = BASE + root_item['href'] if tvdb_name not in root_dict.keys(): root_dict[tvdb_name] = season_url else: root_dict[tvdb_name] = root_dict[tvdb_name] + ',' + season_url for root_name, season_url in root_dict.iteritems(): _common.add_show(root_name, SITE, 'seasons', season_url) _common.set_view('tvshows')
def rootlist(): root_doubles = [] root_dict = {} root_data = _connection.getURL(SHOWS) root_menu = BeautifulSoup(root_data, 'html.parser').find_all('div', class_ = 'title') for root_item in root_menu: root_name = root_item.text.strip() if root_name not in root_doubles: tvdb_name = _common.get_show_data(root_name,SITE, 'seasons')[-1] if tvdb_name not in root_dict.keys(): root_dict[tvdb_name] = root_name else: root_dict[tvdb_name] = root_dict[tvdb_name] + ',' + root_name root_doubles.append(root_name) for root_name in root_dict: season_url = root_dict[root_name] _common.add_show(root_name, SITE, 'seasons', season_url) _common.set_view('tvshows')
def masterlist(): master_dict = {} master_db = [] master_data = _connection.getURL(SHOWS) master_tree = BeautifulSoup(master_data) master_section = master_tree.find_all('div', class_ = 'primetime_and_originals') for section in master_section: master_menu = section.find_all('a', text = True) for master_item in master_menu: master_name = master_item.text tvdb_name = _common.get_show_data(master_name,SITE, 'seasons')[-1] season_url = BASE + master_item['href'] if tvdb_name not in master_dict.keys(): master_dict[tvdb_name] = season_url else: master_dict[tvdb_name] = master_dict[tvdb_name] + ',' + season_url for master_name, season_url in master_dict.iteritems(): master_db.append((master_name, SITE, 'seasons', season_url)) return master_db
def masterlist(): master_db = [] master_doubles = [] master_dict = {} master_data = _connection.getURL(SHOWS) master_menu = BeautifulSoup(master_data, 'html.parser').find_all('div', class_ = 'title') for master_item in master_menu: master_name = master_item.text.strip() if master_name not in master_doubles: tvdb_name = _common.get_show_data(master_name,SITE, 'seasons')[-1] if tvdb_name not in master_dict.keys(): master_dict[tvdb_name] = master_name else: master_dict[tvdb_name] = master_dict[tvdb_name] + ',' + master_name master_doubles.append(master_name) for master_name in master_dict: season_url = master_dict[master_name] master_db.append((master_name, SITE, 'seasons', season_url)) return master_db
def masterlist(): master_db = [] master_doubles = [] master_dict = {} master_data = _connection.getURL(SHOWS) master_menu = BeautifulSoup(master_data, 'html.parser').find_all('div', class_='title') for master_item in master_menu: master_name = master_item.text.strip() if master_name not in master_doubles: tvdb_name = _common.get_show_data(master_name, SITE, 'seasons')[-1] if tvdb_name not in master_dict.keys(): master_dict[tvdb_name] = master_name else: master_dict[ tvdb_name] = master_dict[tvdb_name] + ',' + master_name master_doubles.append(master_name) for master_name in master_dict: season_url = master_dict[master_name] master_db.append((master_name, SITE, 'seasons', season_url)) return master_db
def masterlist(): master_dict = {} master_db = [] master_data = _connection.getURL(SHOWS) master_tree = BeautifulSoup(master_data) master_section = master_tree.find_all('div', class_='primetime_and_originals') for section in master_section: master_menu = section.find_all('a', text=True) for master_item in master_menu: master_name = master_item.text tvdb_name = _common.get_show_data(master_name, SITE, 'seasons')[-1] season_url = BASE + master_item['href'] if tvdb_name not in master_dict.keys(): master_dict[tvdb_name] = season_url else: master_dict[ tvdb_name] = master_dict[tvdb_name] + ',' + season_url for master_name, season_url in master_dict.iteritems(): master_db.append((master_name, SITE, 'seasons', season_url)) return master_db