### Get the current time if we don't already have one (and transform into a date object)
curr_time = scraperfunctions.get_curr_time()
### Establish our MySQL Connection (for logging, etc.)
engine, connection, metadata, mysql_table_name, mysql_log_name = scraperfunctions.create_mysql_engine(
config)
########### Download actions
try:
### Initiate our virtual display
print("Initiating virtual display")
display = Display(visible=0, size=(1920, 1080))
display.start()
### Let's start our browser
browser = scraperfunctions.create_browser()
### Let's load the page work
scraperfunctions.load_homepage(browser, pubshort, puburl)
### See if the MV list requires extra actions
if puburl_mv_extraactions is not None:
browser.find_element_by_xpath(
"id('river_nav_inner')//li[@data-value='popular']").click()
pass
### Let's first store the source code
html_code = browser.page_source
write_out_file = scraperfunctions.write_out_file(
"%s" % homepages_dir,
"%s_%s.html" % (pubshort, curr_time.strftime("%Y%m%d%H%M")), html_code)
curr_time = scraperfunctions.get_curr_time(curr_time, parsefile)
### Establish our MySQL Connection (for logging, etc.)
engine, connection, metadata, mysql_table_name, mysql_log_name = scraperfunctions.create_mysql_engine(
config)
########### Download actions
if download_desktop == 1:
try:
### Initiate our virtual display
print("Initiating virtual display")
display = Display(visible=0, size=(1920, 1080))
display.start()
### Let's start our browser
browser = scraperfunctions.create_browser()
### Let's load the page work
scraperfunctions.load_homepage(browser, pubshort, puburl)
### See if the MV list requires extra actions
if puburl_mv_extraactions != None:
### Actions for acquiring MV List
pass
### Let's first store the source code
html_code = browser.page_source
write_out_file = scraperfunctions.write_out_file(
"%s" % (homepages_dir),
"%s_%s.html" % (pubshort, curr_time.strftime("%Y%m%d%H%M")),
html_code)
### Get the current time if we don't already have one (and transform into a date object)
curr_time = scraperfunctions.get_curr_time(curr_time, parsefile)
### Establish our MySQL Connection (for logging, etc.)
engine, connection, metadata, mysql_table_name, mysql_log_name = scraperfunctions.create_mysql_engine(config)
########### Download actions
if download_desktop == 1:
try:
### Initiate our virtual display
print("Initiating virtual display")
display = Display(visible=0, size=(1920, 1080))
display.start()
### Let's start our browser
browser = scraperfunctions.create_browser()
### Let's load the page work
scraperfunctions.load_homepage(browser, pubshort, puburl)
### See if the MV list requires extra actions
if puburl_mv_extraactions != None:
### Actions for acquiring MV List
pass
### Let's first store the source code
html_code = browser.page_source
write_out_file = scraperfunctions.write_out_file("%s" % (homepages_dir), "%s_%s.html" % (pubshort, curr_time.strftime("%Y%m%d%H%M")), html_code)
### See if the MV list is in a separate URL
if puburl_mv != None:
