root_folder = "../../" homepages_dir = root_folder + "data/html_src/" + pubshort + "/" screenshots_dir = root_folder + "data/screenshots/" + pubshort + "/" ########### Load libraries from selenium import webdriver from datetime import datetime from pyvirtualdisplay import Display import scraperfunctions import time, sys ### Grab the information from our configuration file config = scraperfunctions.load_config() ### Get the current time if we don't already have one (and transform into a date object) curr_time = scraperfunctions.get_curr_time() ### Establish our MySQL Connection (for logging, etc.) engine, connection, metadata, mysql_table_name, mysql_log_name = scraperfunctions.create_mysql_engine( config) ########### Download actions try: ### Initiate our virtual display print("Initiating virtual display") display = Display(visible=0, size=(1920, 1080)) display.start() ### Let's start our browser browser = scraperfunctions.create_browser()
########### Let's check for command-line arguments parser = argparse.ArgumentParser() parser = scraperfunctions.add_common_arguments(parser) ### Add additional arguments here. args = parser.parse_args() curr_time = args.curr_time download_desktop = args.download_desktop download_mobile = args.download_mobile parse = args.parse parsefile = args.parsefile ### Grab the information from our configuration file config = scraperfunctions.load_config() ### Get the current time if we don't already have one (and transform into a date object) curr_time = scraperfunctions.get_curr_time(curr_time, parsefile) ### Establish our MySQL Connection (for logging, etc.) engine, connection, metadata, mysql_table_name, mysql_log_name = scraperfunctions.create_mysql_engine( config) ########### Download actions if download_desktop == 1: try: ### Initiate our virtual display print("Initiating virtual display") display = Display(visible=0, size=(1920, 1080)) display.start() ### Let's start our browser browser = scraperfunctions.create_browser()
########### Let's check for command-line arguments parser = argparse.ArgumentParser() parser = scraperfunctions.add_common_arguments(parser) ### Add additional arguments here. args = parser.parse_args() curr_time = args.curr_time download_desktop = args.download_desktop download_mobile = args.download_mobile parse = args.parse parsefile = args.parsefile ### Grab the information from our configuration file config = scraperfunctions.load_config() ### Get the current time if we don't already have one (and transform into a date object) curr_time = scraperfunctions.get_curr_time(curr_time, parsefile) ### Establish our MySQL Connection (for logging, etc.) engine, connection, metadata, mysql_table_name, mysql_log_name = scraperfunctions.create_mysql_engine(config) ########### Download actions if download_desktop == 1: try: ### Initiate our virtual display print("Initiating virtual display") display = Display(visible=0, size=(1920, 1080)) display.start() ### Let's start our browser browser = scraperfunctions.create_browser()