def __init__(self, browser='chrome'):
if browser == 'chrome':
chrome_options = Options()
chrome_options.add_experimental_option("prefs", {'profile.managed_default_content_settings.javascript': 2})
# https://stackoverflow.com/a/44771628
caps = DesiredCapabilities().CHROME
# caps["pageLoadStrategy"] = "normal" # complete load
# caps["pageLoadStrategy"] = "eager" # interactive
caps["pageLoadStrategy"] = "none"
self.driver = webdriver.Chrome(desired_capabilities=caps, chrome_options=chrome_options)
else:
profile = webdriver.FirefoxProfile()
profile.set_preference("javascript.enabled", False)
caps = DesiredCapabilities().FIREFOX
# caps["pageLoadStrategy"] = "normal" # complete load
caps["pageLoadStrategy"] = "eager" # interactive
# caps["pageLoadStrategy"] = "none"
self.driver = webdriver.Firefox(profile=profile, desired_capabilities=caps)
self.url_base = 'https://www.worldometers.info/coronavirus/country/%s/'
self.country = None
def create_driver(self):
if 1:
caps = DesiredCapabilities().FIREFOX.copy()
profile_path = path.expanduser(
'~') + '/.mozilla/firefox/' + self.account['name']
# caps['proxy'] = {
caps['moz:firefoxOptions'] = {
"args": ["-profile", profile_path], # geckodriver 0.18+
}
profile = FirefoxProfile(profile_path)
#profile.set_preference("general.useragent.override", 'Mozilla/5.0 (X11; Linux x86_64; rv:56.0) Gecko/20100101 Firefox/56.0')
self.driver = Firefox(profile, capabilities=caps)
#self.driver = Firefox(profile)
else: # PhantomJS
# https://github.com/detro/ghostdriver
caps = DesiredCapabilities().PHANTOMJS
caps["phantomjs.page.settings.userAgent"] = \
'Mozilla/5.0 (X11; Linux x86_64; rv:56.0) Gecko/20100101 Firefox/56.0'
service_args = [
'--proxy={}'.format(':'.join(
self.account['Proxy'].split(':')[:2])),
'--proxy-type=http',
]
print(service_args)
self.driver = PhantomJS(service_args=service_args,
capabilities=caps)
self.driver.set_window_size(1120, 550)
def __init__(self, browser="chrome", headless=True, slowdown=0):
# Used so I can see steps taken
self.slowdown = slowdown
if (browser == "chrome"):
self.capabilities = DesiredCapabilities().CHROME
options = Options()
if (headless):
options.add_argument('--headless')
options.add_argument('--disable-gpu')
self.driver = webdriver.Chrome("webdrivers/chromedriver_linux64",
chrome_options=options)
elif (browser == "firefox"):
self.capabilities = DesiredCapabilities().FIREFOX
# self.capabilities["pageLoadStrategy"] = "eager"
# self.capabilities["pageLoadStrategy"] = "normal"
self.driver = webdriver.Firefox(
desired_capabilities=self.capabilities)
else:
logger.error("browser should be either chrome or firefox")
raise Exception("Invalid browser kind supplied to "
"InternetChecker: {} not in {}".format(
browser, self.browser_kinds))
self.webdriver_open = True
logger.debug("Returning from InternetChecker init")
return
def openBrowser(driverPath,defaultUrl,driverOptions):
options = Options()
for option in driverOptions: options.add_argument(option)
if 'geckodriver' in driverPath: browser=webdriver.Firefox(capabilities=DesiredCapabilities().FIREFOX,executable_path=r''+driverPath)
elif 'chromedriver' in driverPath: browser=webdriver.Chrome(desired_capabilities=DesiredCapabilities().CHROME,executable_path=r''+driverPath,chrome_options=options)
browser.get(defaultUrl)
return browser
def __setBrowser(self):
try:
caps = DesiredCapabilities().CHROME.copy()
caps["pageLoadStrategy"] = "eager"
self.__browser = webdriver.Chrome(capabilities=caps)
except:
caps = DesiredCapabilities().FIREFOX.copy()
caps["pageLoadStrategy"] = "eager"
self.__browser = webdriver.Firefox(capabilities=caps)
def _load_driver(self, caps='none', accept_insecure=False):
options = webdriver.ChromeOptions()
prefs = {}
prefs['profile.default_content_settings.popups'] = 0
prefs['download.default_directory'] = self.download_path
prefs['profile.default_content_setting_values.automatic_downloads'] = 1
options.add_experimental_option('prefs', prefs)
options.add_experimental_option("excludeSwitches",
['enable-automation'])
options.add_experimental_option('useAutomationExtension', False)
options.add_argument('--no-sandbox')
options.add_argument('--disable-setuid-sandbox')
options.add_argument('--start-maximized')
if accept_insecure:
options.add_argument('--ignore-ssl-errors=yes')
options.add_argument('--ignore-certificate-errors')
user_agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.53 Safari/537.36'
options.add_argument(f'user-agent={user_agent}')
if not self.head:
options.add_argument('--headless')
try:
if caps == 'none':
caps = DesiredCapabilities().CHROME
caps["pageLoadStrategy"] = "none"
driver = webdriver.Chrome(
executable_path=self._get_chromedriver(),
options=options,
desired_capabilities=caps) # chromedriver
else:
caps = DesiredCapabilities().CHROME
caps["pageLoadStrategy"] = "normal"
driver = webdriver.Chrome(
executable_path=self._get_chromedriver(),
options=options,
desired_capabilities=caps) # chromedriver
driver.execute_script(
f"var s=window.document.createElement('script'); s.src='{self._get_chromedriver_path}javascriptM.js';window.document.head.appendChild(s);"
)
driver.set_window_size(1400, 1000)
driver.set_page_load_timeout(600)
driver.delete_all_cookies()
except Exception as e:
print('Failed to start driver: ' + str(e))
if 'chrome not reachable' in str(e):
print('Try turning off your firewall...')
return driver
def browser_driver():
if BROWSER_VENDOR == "Firefox":
cap = DesiredCapabilities().FIREFOX
# cap["marionette"] = False
return webdriver.Firefox(capabilities=cap,
executable_path=BROWSER_DRIVER)
elif BROWSER_VENDOR == "Chrome":
cap = DesiredCapabilities().CHROME
cap["marionette"] = False
return webdriver.Chrome(executable_path=BROWSER_DRIVER)
else:
raise NameError("'BROWSER_VENDOR' is not defined correctly!")
def driver(config):
driver = None
if config['selenoid'] is not None:
capabilities = {
"browserName": "chrome",
"browserVersion": "80.0",
"selenoid:options": {
"enableVNC": False,
"enableVideo": False
}
}
selenoid_url = 'http://' + config['selenoid'] + '/wd/hub'
options = ChromeOptions()
driver = webdriver.Remote(command_executor=selenoid_url,
options=options,
desired_capabilities=capabilities)
driver.file_detector = LocalFileDetector()
else:
options = ChromeOptions()
driver = webdriver.Chrome(
ChromeDriverManager().install(),
options=options,
desired_capabilities=DesiredCapabilities().CHROME)
driver.maximize_window()
driver.get(config['url'])
yield driver
driver.quit()
def start_browser(link,cookies):
caps = DesiredCapabilities().CHROME
caps["pageLoadStrategy"] = "eager"
chrome_options = ChromeOptions()
chrome_options.add_experimental_option("excludeSwitches", ["enable-automation"])
chrome_options.add_experimental_option("useAutomationExtension", False)
driver = Chrome(desired_capabilities=caps, executable_path=driver_path, options=chrome_options)
driver.execute_cdp_cmd(
"Page.addScriptToEvaluateOnNewDocument",
{
"source": """
Object.defineProperty(window, 'navigator', {
value: new Proxy(navigator, {
has: (target, key) => (key === 'webdriver' ? false : key in target),
get: (target, key) =>
key === 'webdriver'
? undefined
: typeof target[key] === 'function'
? target[key].bind(target)
: target[key]
})
})
"""
},
)
driver.get(link)
for cookie in cookies:
driver.add_cookie({
"name": cookie["name"],
"value" : cookie["value"],
"domain" : cookie["domain"]
})
driver.get(link)
def open_browser(self):
caps = DesiredCapabilities().CHROME
# caps["pageLoadStrategy"] = "normal" # complete
# caps["pageLoadStrategy"] = "eager" # interactive
caps["pageLoadStrategy"] = "none"
self.browser = webdriver.Chrome(chrome_options=self.chrome_options,
desired_capabilities=caps)
def main():
# Options for Chrome WebDriver
caps = DesiredCapabilities().CHROME
caps["pageLoadStrategy"] = "none"
op = Options()
op.add_argument("--disable-notifications")
op.add_experimental_option(
"prefs", {
"download.prompt_for_download": False,
"download.directory_upgrade": True,
"safebrowsing.enabled": True
})
with webdriver.Chrome(desired_capabilities=caps, options=op) as driver:
# Setting Chrome to trust downloads
driver.command_executor._commands["send_command"] = (
"POST", "/session/$sessionId/chromium/send_command")
params = {
"cmd": "Page.setDownloadBehavior",
"params": {
"behavior": "allow",
"downloadPath": "tmp"
}
}
command_result = driver.execute("send_command", params)
driver.get(SOURCE_URL)
time.sleep(10)
links = driver.find_elements_by_css_selector(".services a")
for link in links:
if "Raw Data" in link.text:
nextcloud = link.get_attribute("href")
break
driver.get(nextcloud)
time.sleep(5)
driver.find_element_by_css_selector(".directDownload a").click()
time.sleep(2)
file = glob("tmp/Thailand*")[0]
df = pd.read_excel(file)
df.loc[:, "Date"] = pd.to_datetime(df["Date"], errors="coerce")
df = df[["Date", "Pos", "Total"]].dropna().sort_values("Date")
df["Positive rate"] = (df.Pos.rolling(7).mean() /
df.Total.rolling(7).mean()).round(3)
df = df.rename(columns={
"Total": "Daily change in cumulative total"
}).drop(columns="Pos")
df = df[df["Daily change in cumulative total"] > 0]
df.loc[:, "Country"] = "Thailand"
df.loc[:, "Source URL"] = SOURCE_URL
df.loc[:, "Source label"] = "Ministry of Public Health"
df.loc[:, "Units"] = "tests performed"
df.loc[:, "Notes"] = pd.NA
df.to_csv("automated_sheets/Thailand.csv", index=False)
os.remove(file)
def login(self):
display = Display(visible=0,size=(1024,768))
display.start()
# 登录
try:
# 安管中心环境使用####
driver = webdriver.Firefox()
except:
# 213环境使用########
cap = DesiredCapabilities().FIREFOX
cap["marionette"] = False
driver = webdriver.Firefox(capabilities=cap)
driver.get('https://twitter.com/login')
time.sleep(3)
driver.find_element_by_xpath('//input[@class="js-username-field email-input js-initial-focus"]').send_keys(self.username)
driver.find_element_by_xpath('//input[@class="js-password-field"]').click()
driver.find_element_by_xpath('//input[@class="js-password-field"]').send_keys(self.password)
driver.find_element_by_xpath('//button[@class="submit EdgeButton EdgeButton--primary EdgeButtom--medium"]').click()
time.sleep(1)
req = requests.Session()
cookies = driver.get_cookies()
for cookie in cookies:
req.cookies.set(cookie['name'],cookie['value'])
headers = {
'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:56.0) Gecko/20100101 Firefox/56.0'
}
return driver,display
def main(name):
binary = r'C:\Program Files\Mozilla Firefox\firefox.exe'
options = Options()
options.set_headless(headless=True)
options.binary = binary
cap = DesiredCapabilities().FIREFOX
cap["marionette"] = True #optional
driver = webdriver.Firefox(
options=options,
capabilities=cap,
executable_path="C:\\Users\\Public\\geckodriver.exe")
driver.implicitly_wait(10)
driver.get("https://grandmaster.report/user/1/" + name)
arr = []
try:
element = WebDriverWait(driver, 20).until(
EC.presence_of_element_located(
(By.CLASS_NAME, "UserActivity_cardContainer__2vOAf")))
elements = driver.find_elements_by_class_name(
"UserActivity_cardContainer__2vOAf")
for el in elements:
span = el.find_elements_by_tag_name("p")[1].text.replace('\n', ' ')
print(span)
arr.append(span)
finally:
driver.quit()
def get_driver(profile=None, headless=False, noimage=False, nonblock=False):
option = webdriver.ChromeOptions()
option.add_argument('disable-infobars')
option.add_argument('--hide-scrollbars')
# for headless
option.add_argument('window-size=1920x1080')
# possible bug?
option.add_argument('--disable-gpu')
# enable redirect
option.add_argument("--disable-web-security")
# disable log
option.add_argument("--log-level=3")
# disable selenium flag
option.add_experimental_option('excludeSwitches', ['enable-automation'])
if profile:
option.add_argument("user-data-dir={}".format(profile))
if headless:
option.add_argument('--headless')
if noimage:
prefs = {"profile.managed_default_content_settings.images": 2}
option.add_experimental_option("prefs", prefs)
caps = DesiredCapabilities().CHROME
if nonblock:
# caps["pageLoadStrategy"] = "normal" # complete
# caps["pageLoadStrategy"] = "eager" # interactive
caps["pageLoadStrategy"] = "none"
# To avoid mysterious bug
option.add_argument(
"--remote-debugging-port={}".format(random.randint(10000, 65535)))
driver = webdriver.Chrome(options=option, desired_capabilities=caps)
return driver
def _setup_browser(self, browser_type=None):
if not __debug__:
from selenium import webdriver
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
if browser_type is None: browser_type = "Chrome"
_f = getattr(webdriver, browser_type)
if browser_type == "Chrome":
capa = DesiredCapabilities().CHROME
capa["pageLoadStrategy"] = "none"
from selenium.webdriver.chrome.options import Options
croptions = Options()
# __string, = ("user-agent=Mozilla/5.0 "
# "(Windows NT 10.0; Win64; x64) "
# "AppleWebKit/537.36 (KHTML, like Geoko) "
# "Chrome/70.0.3538.102 Safari/537.36", )
# croptions.add_argument(__string)
browser = _f(
executable_path="/usr/lib/chromium-browser/chromedriver",
desired_capabilities=capa,
chrome_options=croptions,
)
else:
browser = _f()
del _f
else:
browser = self.fakeRM.acquire_browser_handler_by_create()
browser.get(self._gen_url())
sleep(5)
self.browser = browser
def init_chrome_driver(self):
""""Initializing chrome drivers"""
# If there is no instance create one
if not self.chrome_driver:
self.first_run = True
print("Initializing chrome Driver")
try:
# Creating chrome options
caps = DesiredCapabilities().CHROME
caps["pageLoadStrategy"] = "eager"
chrome_options = webdriver.ChromeOptions()
# prefs = {"profile.default_content_setting_values.cookies": 2,
# "profile.block_third_party_cookies": True,
prefs = {
"profile.managed_default_content_settings.images": 2,
"profile.default_content_settings.images": 2
}
chrome_options.add_experimental_option("prefs", prefs)
chrome_options.add_argument('--incognito')
chrome_options.add_argument('--headless')
chrome_options.add_argument('--log-level=3')
chrome_options.add_argument('--silent')
self.chrome_driver = webdriver.Chrome(
executable_path=self.chrome_driver_location,
options=chrome_options)
except Exception as e:
raise ConnectionError("Couldn't init chrome drivers" + str(e))
def build_driver():
cap = DesiredCapabilities().FIREFOX
if OS == "OSX":
executable_path = "./drivers/geckodriver"
elif OS == "Windows":
executable_path = "./drivers/geckodriver.exe"
return webdriver.Firefox(capabilities=cap, executable_path=executable_path)
def main(name):
binary = r'C:\Program Files\Mozilla Firefox\firefox.exe'
options = Options()
options.set_headless(headless=True)
options.binary = binary
cap = DesiredCapabilities().FIREFOX
cap["marionette"] = True #optional
driver = webdriver.Firefox(options=options, capabilities=cap, executable_path="C:\\Users\\Public\\geckodriver.exe")
driver.implicitly_wait(10)
driver.get("https://raid.report/xb/" + name)
arr = [];
try:
element = WebDriverWait(driver, 20).until(
EC.presence_of_element_located((By.CLASS_NAME, "total-completions"))
)
time.sleep(3)
elements = driver.find_elements_by_class_name("total-completions")
elements2 = driver.find_elements_by_class_name("s4")
counter = 0;
for el in elements:
span = el.find_element_by_tag_name('span').text;
arr.append(span);
a = elements2[counter].find_elements_by_tag_name("a")
if len(a) > 0:
arr.append(a[0].text);
else:
arr.append('N/A')
counter = counter + 6;
finally:
driver.quit();
for item in arr:
print(item);
def flush(browser, n):
ua = DesiredCapabilities().IPHONE
for i in range(n):
if browser.lower() == 'firefox':
driver = webdriver.Firefox()
elif browser.lower() == 'chrome':
options = webdriver.ChromeOptions()
options.add_argument('--disable-extensions')
driver = webdriver.Chrome(executable_path=driver_path +
'chromedriver.exe',
chrome_options=options)
elif browser.lower() == 'ie':
driver = webdriver.Ie(executable_path=driver_path +
'IEDriverServer.exe')
elif browser.lower() == 'phantomjs':
killphantomjs()
driver = webdriver.PhantomJS(executable_path=driver_path +
'phantomjs.exe',
desired_capabilities=ua)
driver.get('http://m.baidu.com')
driver.find_element_by_id('index-kw').send_keys(
random.choice(KEYWORDS), Keys.ENTER)
clk(driver, url='csdn')
sleep(1)
print driver.find_element_by_class_name('article_t').text,
print driver.find_element_by_xpath('//p[@class="date"]/i[2]').text
driver.close()
def get_image(chart, lib):
''' Gets chart as image '''
if lib == 'BOKEH':
img = get_screenshot_as_png(chart)
roi_img = img.crop()
img_byte_array = io.BytesIO()
roi_img.save(img_byte_array, format='PNG')
return img_byte_array.getvalue()
if lib == 'FOLIUM':
ff_options = Options()
ff_options.set_headless(headless=True)
cap = DesiredCapabilities().FIREFOX
cap["marionette"] = True
driver = webdriver.Firefox(firefox_options=ff_options, capabilities=cap)
driver.get(
"data:text/html;charset=utf-8,{html_content}".format(
html_content=chart._repr_html_()
)
)
return driver.get_screenshot_as_png()
return None
def before_all(context):
print("Cleanup Logs and Snapshots Folders")
files = glob.glob('./Logs/*')
for f in files:
os.remove(f)
files = glob.glob('./Screenshots/*')
for f in files:
os.remove(f)
if (configReader.readConfigData('Details', 'Browser')) == "chrome":
path = "./Drivers/chromedriver.exe"
context.driver = webdriver.Chrome(executable_path=path)
elif (configReader.readConfigData('Details', 'Browser')) == "firefox":
path = "./Drivers/geckodriver.exe"
cap = DesiredCapabilities().FIREFOX
cap["marionette"] = False
context.driver = webdriver.Firefox(capabilities=cap, executable_path=path)
elif (configReader.readConfigData('Details', 'Browser')) == "ie":
path = "./Drivers/IEDriverServer.exe"
context.driver = webdriver.Ie(executable_path=path)
else:
path = "./Drivers/chromedriver.exe"
context.driver = webdriver.Chrome(executable_path=path)
context.driver.get(configReader.readConfigData('Details', 'IP'))
context.driver.maximize_window()
context.driver.implicitly_wait(10)
context.driver.maximize_window()
def get_driver(self, driver=DEFAULT_DRIVER) -> RemoteWebDriver:
"""
Funcao para requisitar um driver especifico
:param driver: Driver que deve ser requisitado
:return: RemoteWebDriver
"""
self.__driver_instance = None
import sys
from os.path import dirname,abspath
from tools.system_architeture import SystemArchiteture
extension = '.exe' if SystemArchiteture.get_operational_system() == 'WindowsPE' else ''
fname = driver+"{0}".format(extension)
path = dirname(abspath(sys.argv[0]))+'/{0}'.format(fname)
if driver == 'geckodriver':
cap = DesiredCapabilities().FIREFOX
self.__driver_instance = webdriver.Firefox(executable_path=path,capabilities=cap)
else:
raise DriverRequestException(u"O driver [{}] não pode ser instanciado.".format(driver))
return self.__driver_instance
def __init__(self, user, password):
self.user = user
self.password = password
self.driver = None
socket.setdefaulttimeout(30)
if platform.system() == 'Linux':
# self.display = Display(visible=1, size=(1024, 768))
# self.display.start()
binary = FirefoxBinary('/usr/bin/firefox')
caps = DesiredCapabilities().FIREFOX.copy()
self.driver = webdriver.Firefox(timeout=30,
capabilities=caps,
executable_path='/usr/bin/geckodriver',
firefox_binary=binary)
else:
self.driver = webdriver.Chrome()
self.driver.set_page_load_timeout(30)
self.driver.get(self.url)
elem = self.driver.find_element_by_name("email")
elem.clear()
elem.send_keys(user)
elem = self.driver.find_element_by_name("pass")
elem.send_keys(password)
elem.send_keys(Keys.RETURN)
timeCycles = 0
timeWait = 2
while "logout" not in self.driver.page_source:
if timeCycles > 5:
print ("Failed to login to page")
break
time.sleep(timeWait)
timeCycles += 1
def selenium_setup():
#DEFINE CHROME CAPABILITIES TO WAIT FOR PAGE TO BE INTERACTIVE INSTEAD OF FULL LOAD
chrome_capabilities = DesiredCapabilities().CHROME
chrome_capabilities["pageLoadStrategy"] = "eager"
#DEFINE CHROME OPTIONS
chrome_options = webdriver.ChromeOptions()
user_agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36'
chrome_options.add_argument(f'user-agent={user_agent}')
chrome_options.add_argument("--lang=en-US,en;q=.9")
chrome_options.add_experimental_option(
"prefs", {"profile.managed_default_content_settings.images": 2})
chrome_options.add_argument("--disable-gpu")
chrome_options.add_argument("--headless")
chrome_options.add_argument("start-maximized")
chrome_options.add_argument("disable-infobars")
chrome_options.add_argument("--incognito")
chrome_options.add_experimental_option('excludeSwitches',
['enable-automation'])
chrome_options.add_experimental_option('useAutomationExtension', False)
#SET SELENIUM CHROME DRIVER CAPABILITIES/OPTIONS/PATH HERE
browser_driver = webdriver.Chrome(
desired_capabilities=chrome_capabilities,
options=chrome_options,
executable_path=
r'C:\Users\Mason\Desktop\DeliverMeScraper\chromedriver.exe')
#RETURN BROWSER
return browser_driver
def main():
url = "https://www.climatempo.com.br/climatologia/2/santos-sp"
chrome_options = Options()
caps = DesiredCapabilities().CHROME
caps["pageLoadStrategy"] = "none"
prefs = {
"profile.default_content_setting_values.notifications": 2,
"profile.default_content_setting_values.geolocation": 2,
"profile.default_content_settings": {"popups": 1},
}
chrome_options.add_experimental_option("prefs", prefs)
chrome_options.add_argument("--headless")
driver = webdriver.Chrome(desired_capabilities=caps, chrome_options=chrome_options)
try:
city_limit = int(sys.argv[1])
logger.info("Using custom city scrapper limit %s", city_limit)
scrape(url, driver, city_limit)
except IndexError:
logger.info("Using default city scrapper limit %s", CITY_LIMIT)
scrape(url, driver, CITY_LIMIT)
def get_driver(browser):
options = Options()
options.headless = True
capabilities = DesiredCapabilities().FIREFOX
# capabilities["pageLoadStrategy"] = "eager"
if browser == 'Firefox':
return webdriver.Firefox(options=options, capabilities=capabilities)
def OpenPageThenSave(page_url, save_to_file):
cap = DesiredCapabilities().FIREFOX
cap["marionette"] = True
options = Options()
options.headless = False
browser = webdriver.Firefox(capabilities=cap, options=options)
browser.set_page_load_timeout(30) # seconds
try:
browser.get(page_url)
except:
print("timeout loading page: ", page_url)
# To simulate a Save As dialog.
pyautogui.hotkey('ctrl', 's')
# Wait for the Save As dialog to load. Might need to increase the wait time on slower machines
time.sleep(2)
# File path + name
# FILE_NAME = 'C:\\path\\to\\file\\file.ext'
# Type the file path and name is Save AS dialog
pyautogui.typewrite(save_to_file)
time.sleep(3)
# Hit Enter to save
pyautogui.hotkey('enter')
pyautogui.hotkey('enter')
time.sleep(5)
browser.quit()
def create_browser(path_chrome_drive):
#--------------------------------------------------------------------------------
# option setting : Chrome w/o GUI (headless mode)
#--------------------------------------------------------------------------------
options = webdriver.ChromeOptions()
options.add_argument('headless')
#options.add_argument('window-size=1920x1080')
#options.add_argument("disable-gpu")
#options.add_argument("--start-maximized")
options.add_experimental_option(
"prefs", {"download.default_directory": os.getcwd()})
#options.add_experimental_option("prefs", { "download.default_directory": def_dir_sav })
#--------------------------------------------------------------------------------
# option setting : Chrome not to wait until full page loading
#--------------------------------------------------------------------------------
caps = DesiredCapabilities().CHROME
#caps["pageLoadStrategy"] = "normal" # complete
caps["pageLoadStrategy"] = "eager" # interactive
#caps["pageLoadStrategy"] = "none" # you need "chrome.implicitly_wait(3)" statement (some site's full loading time is so long)
#--------------------------------------------------------------------------------
# Launch Chrome Driver
#--------------------------------------------------------------------------------
ch = webdriver.Chrome(
path_chrome_drive, desired_capabilities=caps,
options=options) # non-def-pageLoadStrategy, Headless(no-GUI)
#ch = webdriver.Chrome(path_chrome_drive, desired_capabilities=caps)
#ch = webdriver.Chrome(path_chrome_drive, options=options)
#ch = webdriver.Chrome(path_chrome_drive)
#ch.implicitly_wait(3) # An implicit wait tells WebDriver to poll the DOM for a certain amount of time when trying to find any element (or elements) not immediately available. The default setting is 0. Once set, the implicit wait is set for the life of the WebDriver object.
return (ch)
def main():
args = init_args()
logging.basicConfig(
level=logging.ERROR,
format='%(asctime)s %(name)s %(levelname)s: %(message)s')
LOG.setLevel(logging.INFO)
LOG.info('Reading rankings')
rankings = read_rankings(args)
capabilities = DesiredCapabilities().FIREFOX
capabilities["pageLoadStrategy"] = "eager"
driver = Firefox(capabilities=capabilities)
driver.implicitly_wait(5)
LOG.info('Loading url %s', URL)
driver.get(URL)
LOG.info('Logging in')
login(driver)
time.sleep(3)
if args.reset and input('Reset current draft order [y/N]: ') in ('y', 'Y'):
LOG.info('Resetting roster')
reset_roster(driver)
LOG.info('Waiting for page to load')
driver.find_elements_by_css_selector('#all_player_list')
add_players(args, driver, rankings)
input('Press any key to exit')
driver.close()
def __init__(self):
self.builtIn = False
self.google = False
self.days = None
self.minimum = None
self.maximum = None
self.contractToHireBoolean = True
self.caps = DesiredCapabilities().CHROME
self.caps["pageLoadStrategy"] = "eager"
self.checkIsNumber = re.compile('^[0-9\,\s]+$')
self.findYearsOfExperience = re.compile(
'([\d{1}][\d{1}]?)-?\s?(to)?\s?([\d{1}][\d{1}]?)?\+?\syear.*(experience|implementing|writing)'
)
self.contractToHire = re.compile(
'(has a client)?((d|D)uration(:)? [\d]?-?[\d]?[\d]? (month(s)?|year(s)?|week(s)?))?((o|O)ur .* client is (looking|seeking|searching))?((C|c)ontract position)?([\d]?-?[\d]?[\d]? (month(s)?|year(s)?|week(s))? contract)?'
)
self.t2d = text2digits.Text2Digits()
self.driver = webdriver.Chrome(
desired_capabilities=self.caps,
executable_path=ChromeDriverManager().install())
if os.path.exists('./utils/jobs.yml') == True:
with open(r'./utils/jobs.yml') as file:
document = yaml.full_load(file)
self.jobs = list(document[0].values())[0]
else:
print("Specific job titles need to be set. Exiting...")
time.sleep(1)
return False
if os.path.exists('./utils/experience.yml') == True:
with open(r'./utils/experience.yml') as f:
data = yaml.load(f, Loader=yaml.FullLoader)
self.minimum = int(
data['minimum']) if data['minimum'] != None else None
self.maximum = int(
data['maximum']) if data['maximum'] != None else None
self.includeNoMention = bool(data['includeNoMention'])
else:
self.minimum = None
self.maximum = None
if os.path.exists('./utils/files.yml') == True:
with open(r'./utils/files.yml') as file:
self.path = yaml.full_load(file)
else:
self.path = {}
if os.path.exists('./utils/keywords.yml') == True:
with open(r'./utils/keywords.yml') as file:
document = yaml.full_load(file)
self.keywords = list(document[0].values())[0]
else:
self.keywords = []
if os.path.exists('./utils/antiwords.yml') == True:
with open(r'./utils/antiwords.yml') as file:
document = yaml.full_load(file)
self.antiwords = list(document[0].values())[0]
else:
self.antiwords = []
