"id('river_nav_inner')//li[@data-value='popular']").click() pass ### Let's first store the source code html_code = browser.page_source write_out_file = scraperfunctions.write_out_file( "%s" % homepages_dir, "%s_%s.html" % (pubshort, curr_time.strftime("%Y%m%d%H%M")), html_code) ### See if the MV list is in a separate URL if puburl_mv is not None: ### Actions for acquiring MV List pass ### Save a screenshot scraperfunctions.take_screenshot(browser, screenshots_dir, pubshort, curr_time.strftime("%Y%m%d%H%M")) print("Screenshot taken") ### Close the browser scraperfunctions.close_browser(browser) ### Close our virtual display display.stop() print("Display closed") ### Perform closing actions print("Successfully downloaded the page!") #scraperfunctions.store_mysql_log(mysql_log_name, metadata, pubshort, curr_time, "1", "1", "") # Log success except: scraperfunctions.store_mysql_log(mysql_log_name, metadata, pubshort, curr_time, "1", "0", "") # Log failure
pass ### Let's first store the source code html_code = browser.page_source write_out_file = scraperfunctions.write_out_file( "%s" % (homepages_dir), "%s_%s.html" % (pubshort, curr_time.strftime("%Y%m%d%H%M")), html_code) ### See if the MV list is in a separate URL if puburl_mv != None: ### Actions for acquiring MV List pass ### Save a screenshot scraperfunctions.take_screenshot(browser, screenshots_dir, pubshort, curr_time.strftime("%Y%m%d%H%M")) print("Screenshot taken") ### Close the browser scraperfunctions.close_browser(browser) ### Close our virtual display display.stop() print("Display closed") ### Perform closing actions print("Successfully downloaded the page!") #scraperfunctions.store_mysql_log(mysql_log_name, metadata, pubshort, curr_time, "1", "1", "") # Log success except: scraperfunctions.store_mysql_log(mysql_log_name, metadata, pubshort, curr_time, "1", "0",
### See if the MV list requires extra actions if puburl_mv_extraactions != None: ### Actions for acquiring MV List pass ### Let's first store the source code html_code = browser.page_source write_out_file = scraperfunctions.write_out_file("%s" % (homepages_dir), "%s_%s.html" % (pubshort, curr_time.strftime("%Y%m%d%H%M")), html_code) ### See if the MV list is in a separate URL if puburl_mv != None: ### Actions for acquiring MV List pass ### Save a screenshot scraperfunctions.take_screenshot(browser, screenshots_dir, pubshort, curr_time.strftime("%Y%m%d%H%M")) print("Screenshot taken") ### Close the browser scraperfunctions.close_browser(browser) ### Close our virtual display display.stop() print("Display closed") ### Perform closing actions print("Successfully downloaded the page!") #scraperfunctions.store_mysql_log(mysql_log_name, metadata, pubshort, curr_time, "1", "1", "") # Log success except: scraperfunctions.store_mysql_log(mysql_log_name, metadata, pubshort, curr_time, "1", "0", "") # Log failure