### Let's first store the source code html_code = browser.page_source write_out_file = scraperfunctions.write_out_file("%s" % (homepages_dir), "%s_%s.html" % (pubshort, curr_time.strftime("%Y%m%d%H%M")), html_code) ### See if the MV list is in a separate URL if puburl_mv != None: ### Actions for acquiring MV List pass ### Save a screenshot scraperfunctions.take_screenshot(browser, screenshots_dir, pubshort, curr_time.strftime("%Y%m%d%H%M")) print("Screenshot taken") ### Close the browser scraperfunctions.close_browser(browser) ### Close our virtual display display.stop() print("Display closed") ### Perform closing actions print("Successfully downloaded the page!") #scraperfunctions.store_mysql_log(mysql_log_name, metadata, pubshort, curr_time, "1", "1", "") # Log success except: scraperfunctions.store_mysql_log(mysql_log_name, metadata, pubshort, curr_time, "1", "0", "") # Log failure if download_mobile == 1: # Grab Mobile Page try: ### Initiate our virtual display
if puburl_mv != None: scraperfunctions.load_homepage( browser, pubshort, "http://www.twincities.com/popular#pop-day") html_code = browser.page_source write_out_file = scraperfunctions.write_out_file( "%s" % (homepages_dir), "%s_poplist_%s.html" % (pubshort, curr_time.strftime("%Y%m%d%H%M")), html_code) pass ### Save a screenshot scraperfunctions.take_screenshot(browser, screenshots_dir, pubshort, curr_time.strftime("%Y%m%d%H%M")) print("Screenshot taken") ### Close the browser scraperfunctions.close_browser(browser) ### Close our virtual display display.stop() print("Display closed") ### Perform closing actions print("Successfully downloaded the page!") #scraperfunctions.store_mysql_log(mysql_log_name, metadata, pubshort, curr_time, "1", "1", "") # Log success except: scraperfunctions.store_mysql_log(mysql_log_name, metadata, pubshort, curr_time, "1", "0", "") # Log failure if download_mobile == 1: # Grab Mobile Page