def scrape_and_write_medias(self): """Gets movies and shows and writes to database. ShoScraper uses lighter requests/bs4 not selenium/chromedriver. """ logging.info('SHOWTIME MOVIE SEARCH') self.get_movie_pages() self.get_movies() self.lookup_and_write_medias(medias=self.movies, mtype='movie') logging.info('SHOWTIME SHOW SEARCH') self.get_shows() self.lookup_and_write_medias(medias=self.shows, mtype='show') # remove any sources not just updated: media this provider no longer has flaskapp.remove_old_sources('showtime')
def scrape_and_write_medias(self): """Gets movies and shows and writes to database.""" self.start_driver(window_size='--window-size=1920,6000') logging.info('HBO MOVIE SEARCH') movies = self.get_medias_from_page('/movies', mtype='movie') movies = self.add_years_to_movies(movies) self.lookup_and_write_medias(medias=movies, mtype='movie') logging.info('HBO SHOW SEARCH') shows = self.get_medias_from_page('/series', mtype='show') self.lookup_and_write_medias(medias=shows, mtype='show') self.stop_driver() # remove any sources not just updated: media this provider no longer has flaskapp.remove_old_sources('hbo')
def scrape_and_write_medias(self): """Gets movies and shows and writes to database.""" self.start_driver() if self.login(): logging.info('HULU MOVIE SEARCH') movie_genre_pages = self.get_genre_pages(mtype='movie') movies = self.get_medias(movie_genre_pages) self.lookup_and_write_medias(medias=movies, mtype='movie') logging.info('HULU SHOW SEARCH') show_genre_pages = self.get_genre_pages(mtype='show') shows = self.get_medias(show_genre_pages) self.lookup_and_write_medias(medias=shows, mtype='show') else: logging.error('HULU login failed') self.stop_driver() flaskapp.remove_old_sources('hulu') # remove sources not just updated flaskapp.remove_hulu_addon_media() # remove overlap of sho and hulu
def scrape_and_write_medias(self): """Gets movies and shows and writes to database.""" self.start_driver() if self.login(): logging.info('NETFLIX SHOW SEARCH') shows = self.get_medias(mtype='show') self.lookup_and_write_medias(medias=shows, mtype='show') logging.info('NETFLIX MOVIE SEARCH') movies = self.get_medias(mtype='movie') # restart driver, effectively logging out, for rate limiting self.stop_driver() self.start_driver() # add years to movies movies = self.add_years_to_movies(movies) self.lookup_and_write_medias(medias=movies, mtype='movie') else: logging.error('NETFLIX login failed') self.stop_driver() # remove any sources not just updated: media this provider no longer has flaskapp.remove_old_sources('netflix')
def search_hulu(): # source dict to be added to media sources[] in db for found titles source = { 'name': 'hulu', 'display_name': 'Hulu', 'link': 'http://www.hulu.com' } # go to hulu splash page driver = webdriver.PhantomJS(service_log_path='log/phantomjs.log') driver.implicitly_wait(10) # seconds driver.set_window_size(1920, 1080) driver.get('https://www.hulu.com') time.sleep(1.2) # click on log in link links = driver.find_elements_by_tag_name('a') links[2].click() time.sleep(1.2) logging.info('hulu, clicked on log in link') # enter credentials and click login button popup = driver.find_element_by_id('login-popup-section') form = popup.find_element_by_tag_name('form') email_input = form.find_elements_by_tag_name('input')[0] pw_input = form.find_elements_by_tag_name('input')[1] email_input.send_keys(creds['hulu_u']) pw_input.send_keys(creds['hulu_p']) logging.info('hulu, pasted u/p') # driver.save_screenshot('static/screenshot.png') # driver.find_element_by_id('recaptcha_response_field').send_keys('') form.find_element_by_tag_name('button').click() time.sleep(1.2) try: # sometimes first click does not work form.find_element_by_tag_name('button').click() except: pass time.sleep(1.2) # switch out of iframe and click profile link driver.find_element_by_id('62038018').click() time.sleep(1.2) logging.info('hulu, clicked profile') driver.save_screenshot('static/screenshot2.png') def get_medias_from_genre_pages(genre_pages): medias = [] for page in genre_pages: if page == 'https://www.hulu.com/videogames': continue if page == 'https://www.hulu.com/latino': continue # says movie genre but shows not movies # get page and pointer to top panel, holding about 6 medias driver.get(page) logging.info('did get on page: {}'.format(page)) time.sleep(8) top_panel = driver.find_element_by_class_name('tray') next_btn = top_panel.find_element_by_class_name('next') next_counter = 0 # get visible media, click next, repeat until no next button while True: thumbnails = top_panel.find_elements_by_class_name('row') for t in thumbnails: try: # get movie year, show first air year not displayed year = t.find_element_by_tag_name('img') year = year.get_attribute('alt') if re.search('\([0-9][0-9][0-9][0-9]\)$', year): year = year[-5:-1] else: year = '' except: year = '' try: title = t.find_element_by_class_name('title') title = title.get_attribute('innerHTML') # logging.info('title in html found: {}'.format(title)) link = t.find_element_by_class_name('beacon-click') link = link.get_attribute('href') medias += [{ 'title': title, 'link': link, 'year': year }] except NoSuchElementException: logging.warning('no title in row html, blank grid') # with open('log/selenium_error_html_dump.txt', # 'w') as f: # f.write(str(driver.page_source)) continue except StaleElementReferenceException: logging.error('missed a title, may need to wait more') continue if not next_btn.is_displayed(): break # exit loop if next button is not displayed next_btn.click() next_counter += 1 if next_counter % 10 == 0: logging.info('clicked next {} times'.format(next_counter)) if next_counter >= 120: logging.error('next button never went away, may have ' + 'not gotten all media on: {}'.format(page)) break # exit loop, pages should never be this long time.sleep(float(random.randrange(1900, 2300, 1)) / 1000) logging.info('len(medias) so far: {}'.format(len(medias))) return medias # MOVIE SEARCH SECTION logging.info('HULU MOVIE SEARCH') driver.get('https://www.hulu.com/movies/genres') time.sleep(1.5) all_genre = driver.find_element_by_id('all_movies_genres') anchors = all_genre.find_elements_by_class_name('beacon-click') genre_pages = [a.get_attribute('href') for a in anchors] logging.info('hulu, got movie genres') medias = get_medias_from_genre_pages(genre_pages) lookup_and_write_medias(medias, mtype='movie', source=source) # SHOW SEARCH SECTION logging.info('HULU SHOW SEARCH') driver.get('https://www.hulu.com/tv/genres') time.sleep(1.5) all_genre = driver.find_element_by_id('all_tv_genres') anchors = all_genre.find_elements_by_class_name('beacon-click') genre_pages = [a.get_attribute('href') for a in anchors] logging.info('hulu, got tv genres') medias = get_medias_from_genre_pages(genre_pages) lookup_and_write_medias(medias, mtype='show', source=source) driver.quit() # remove any sources not just updated: media this provider no longer has flaskapp.remove_old_sources('hulu')
def search_hbo(): driver = webdriver.PhantomJS(service_log_path='log/phantomjs.log') driver.implicitly_wait(10) # seconds driver.set_window_size(1920, 15000) base_url = 'https://play.hbogo.com' source = {'name': 'hbo', 'display_name': 'HBO', 'link': base_url} pages = [{ 'url': '/movies', 'mtype': 'movie' }, { 'url': '/series', 'mtype': 'show' }, { 'url': '/documentaries', 'mtype': 'movie' }] for page in pages: logging.info('HBO SEARCH OF ' + page['url']) driver.get(base_url + page['url']) time.sleep(5) driver.execute_script("window.scrollTo(0, 10000);") time.sleep(15) # get all boxes with media image and text boxes = driver.find_elements_by_xpath( "//a[@class='default class2 class4']") logging.info(u'num of media boxes found: {}'.format(len(boxes))) # create list of titles and links, replacing newline medias = [] for i, b in enumerate(boxes): title = b.text.replace('\n', ' ') medias += [{'title': title, 'link': b.get_attribute('href')}] # remove non-media, TODO make not catch false positives medias = [m for m in medias if not m['title'].isupper()] # get year if not already in database logging.info('getting year for all media not in database') for m in medias: if page['mtype'] == 'movie' and not flaskapp.db_lookup_via_link( m['link']): driver.get(m['link']) time.sleep(float(random.randrange(5000, 10000, 1)) / 1000) texts = driver.find_element_by_tag_name("body").text texts = texts.split('\n') years = [t for t in texts if re.search('^\d{4}.+min$', t)] if len(years) > 0: m['year'] = years[0][:4] logging.info('year lookup: {}: {}'.format( m['title'], m.get('year', ''))) lookup_and_write_medias(medias, mtype=page['mtype'], source=source) driver.quit() # remove any sources not just updated: media this provider no longer has flaskapp.remove_old_sources('hbo')
def search_showtime(): # source dict to be added to media sources[] in db for found titles base_url = 'http://www.sho.com' source = {'name': 'showtime', 'display_name': 'Showtime', 'link': base_url} # MOVIE SEARCH SECTION logging.info('SHOWTIME MOVIE SEARCH') r = requests.get(base_url + '/movies') soup = BeautifulSoup(r.text, 'html.parser') # get all movie genre pages full_mov_lib = soup.find('section', {'data-context': 'slider:genres'}) genre_links = full_mov_lib.find_all('a', {'class': 'promo__link'}) genre_links = [a['href'] for a in genre_links] genre_links = [i for i in genre_links if 'adult' not in i] # for all root genre pages, get extra pagination links to scrape all_extra_pages = [] for link in genre_links: r = requests.get(base_url + link) soup = BeautifulSoup(r.text, 'html.parser') extra_pages = soup.find('ul', 'pagination__list') if extra_pages: extra_pages = extra_pages.find_all('a') extra_pages = [a['href'] for a in extra_pages] all_extra_pages.extend(extra_pages) genre_links.extend(all_extra_pages) # for all root and extra genre pages, get movie titles catalog = [] for link in genre_links: r = requests.get(base_url + link) logging.info('did get on page: {}'.format(link)) soup = BeautifulSoup(r.text, 'html.parser') anchors = soup.find_all('a', {'class': 'movies-gallery__item'}) for a in anchors: title = a['data-label'] title = title[title.find(':') + 1:] catalog += [{'title': title, 'link': base_url + a['href']}] logging.info('will now check avail on {} catalog items'.format( len(catalog))) # check availability via link, build medias list medias = [] for i, c in enumerate(catalog): time.sleep(0.100) r = requests.get(c['link']) soup = BeautifulSoup(r.text, 'html.parser') year = soup.find_all('dd')[-1].text if year and re.search('^\d{4}$', year): c['year'] = year if soup.find(text='STREAM THIS MOVIE'): medias += [c] if i % 100 == 0: logging.info(u'checked availability on {} items'.format(i)) lookup_and_write_medias(medias, mtype='movie', source=source) # SHOW SEARCH SECTION logging.info('SHOWTIME SHOW SEARCH') r = requests.get(base_url + '/series') soup = BeautifulSoup(r.text, 'html.parser') all_series = soup.find('section', {'data-context': 'promo group:All Showtime Series'}) # get all show titles medias = [] anchors = all_series.find_all('a', {'class': 'promo__link'}) for a in anchors: title = a.text.strip() link = base_url + a['href'] medias += [{'title': title, 'link': link}] lookup_and_write_medias(medias, mtype='show', source=source) # remove any sources not just updated: media this provider no longer has flaskapp.remove_old_sources('showtime')
def search_netflix(): # source dict to be added to media sources[] in db for found titles base_url = 'http://www.netflix.com' source = {'name': 'netflix', 'display_name': 'Netflix', 'link': base_url} # log in to provider driver = webdriver.PhantomJS(service_log_path='log/phantomjs.log') driver.implicitly_wait(10) # seconds driver.set_window_size(1920, 1080) driver.get('https://www.netflix.com/login') inputs = driver.find_elements_by_tag_name('input') inputs[0].send_keys(creds['nf_u']) inputs[1].send_keys(creds['nf_p']) driver.find_element_by_tag_name('button').click() logging.info('netflix, logged in') def get_medias_from_genre_pages(genre_pages): medias = [] for page in genre_pages: # get page and scroll to bottom many times time.sleep(1.5) driver.get(page + '?so=su') logging.info('did get on page: {}'.format(page)) for i in range(40): driver.execute_script( "window.scrollTo(0, document.body.scrollHeight);") time.sleep(float(random.randrange(900, 1400, 1)) / 1000) # put source into beautifulsoup and get titles soup = BeautifulSoup(driver.page_source, 'html.parser') divs = soup('div', 'ptrack-content') for d in divs: title = d.find('div', 'video-preload-title-label').text elements = d['data-ui-tracking-context'].split(',') vid_element = [i for i in elements if 'video_id' in i] netflix_id = vid_element[0][vid_element[0].find(':') + 1:] link = base_url + '/title/' + netflix_id medias += [{'title': title, 'link': link}] logging.info('len(medias) so far: {}'.format(len(medias))) return medias # MOVIE SEARCH SECTION logging.info('NETFLIX MOVIE SEARCH') genre_pages = [ 'https://www.netflix.com/browse/genre/5977', # gay 'https://www.netflix.com/browse/genre/1365', # action 'https://www.netflix.com/browse/genre/5763', # drama 'https://www.netflix.com/browse/genre/7077', # indie 'https://www.netflix.com/browse/genre/8711', # horror 'https://www.netflix.com/browse/genre/6548', # comedy 'https://www.netflix.com/browse/genre/31574', # classics 'https://www.netflix.com/browse/genre/7424', # anime 'https://www.netflix.com/browse/genre/783', # kid 'https://www.netflix.com/browse/genre/7627', # cult 'https://www.netflix.com/browse/genre/6839', # docs ~1321 'https://www.netflix.com/browse/genre/78367', # internat'l 'https://www.netflix.com/browse/genre/8883', # romance 'https://www.netflix.com/browse/genre/1492', # scifi 'https://www.netflix.com/browse/genre/8933' # thrillers ] medias = get_medias_from_genre_pages(genre_pages) medias = get_netflix_year(medias) lookup_and_write_medias(medias, mtype='movie', source=source) # SHOW SEARCH SECTION logging.info('NETFLIX SHOW SEARCH') genre_pages = [ 'https://www.netflix.com/browse/genre/83', # tv popular 'https://www.netflix.com/browse/genre/10673', # action 'https://www.netflix.com/browse/genre/10375', # com 'https://www.netflix.com/browse/genre/11714', # drama 'https://www.netflix.com/browse/genre/83059', # horror 'https://www.netflix.com/browse/genre/4366', # mystery 'https://www.netflix.com/browse/genre/52780', # sci 'https://www.netflix.com/browse/genre/4814', # miniseries 'https://www.netflix.com/browse/genre/46553' # classic ] medias = get_medias_from_genre_pages(genre_pages) lookup_and_write_medias(medias, mtype='show', source=source) driver.quit() # remove any sources not just updated: media this provider no longer has flaskapp.remove_old_sources('netflix')