def driver_factory(browser, executor):
if browser == "chrome":
logger = logging.getLogger('chrome_fixture')
logger.setLevel(LOG_LEVEL)
options = ChromeOptions()
options.headless = True
options.add_argument('--ignore-ssl-errors=yes')
options.add_argument('--ignore-certificate-errors')
logger.info("Подготовка среды для запуска тестов...")
options.add_experimental_option('w3c', False)
driver = EventFiringWebDriver(
webdriver.Remote(command_executor=f"http://{executor}:4444/wd/hub",
desired_capabilities={
"browserName": browser,
"platform": "WIN10",
"platformName": "WIN10"
},
options=options), MyListener())
logger.debug(
"Браузер Chrome запущен со следующими desired_capabilities:{}".
format(driver.desired_capabilities))
elif browser == "firefox":
profile = FirefoxProfile()
profile.accept_untrusted_certs = True
options = FirefoxOptions()
options.headless = True
driver = webdriver.Firefox(options=options, firefox_profile=profile)
else:
raise Exception("Driver not supported")
return driver
def driver_factory(browser):
if browser == "chrome":
logger = logging.getLogger('chrome_fixture')
logger.setLevel(LOG_LEVEL)
options = ChromeOptions()
options.headless = True
options.add_argument('--ignore-ssl-errors=yes')
options.add_argument('--ignore-certificate-errors')
logger.info("Подготовка среды для запуска тестов...")
caps = DesiredCapabilities.CHROME
caps['loggingPrefs'] = {'performance': 'ALL', 'browser': 'ALL'}
options.add_experimental_option('w3c', False)
driver = EventFiringWebDriver(
webdriver.Chrome(desired_capabilities=caps, options=options),
MyListener())
logger.debug(
"Браузер Chrome запущен со следующими desired_capabilities:{}".
format(driver.desired_capabilities))
elif browser == "firefox":
profile = FirefoxProfile()
profile.accept_untrusted_certs = True
options = FirefoxOptions()
options.headless = True
driver = webdriver.Firefox(options=options, firefox_profile=profile)
else:
raise Exception("Driver not supported")
return driver
def driver_factory(browser):
if browser == "chrome":
options = ChromeOptions()
options.headless = True
driver = webdriver.Chrome(options=options)
elif browser == "firefox":
options = FirefoxOptions()
options.headless = True
driver = webdriver.Firefox(options=options)
else:
raise Exception("Driver not supported")
return driver
def process_request(self, request, spider):
options = ChromeOptions()
options.headless = True
driver = Chrome(options=options)
driver.implicitly_wait(20)
driver.get('https://gaudiy.com/community_details/avJEInz3EXlxNXKMSWxR')
time.sleep(0.3)
input_element = driver.find_elements_by_css_selector('span:nth-child(5) > button > span > p')[0]
if input_element:
input_element.click()
time.sleep(0.3)
nft_element = driver.find_elements_by_css_selector('span.MuiTab-wrapper')[0]
if nft_element:
nft_element.click()
source_element = driver.find_element_by_css_selector('label.MuiFormControlLabel-root')
if source_element:
# source_element.click()
time.sleep(1.0)
link = driver.find_elements_by_css_selector('button > div > p:nth-child(1)')[-2]
driver.execute_script("arguments[0].scrollIntoView(true);", link)
time.sleep(0.3)
while link != driver.find_elements_by_css_selector('button > div > p:nth-child(1)')[-2]:
link = driver.find_elements_by_css_selector('button > div > p:nth-child(1)')[-2]
driver.execute_script("arguments[0].scrollIntoView(true);", link)
time.sleep(0.3)
return HtmlResponse(
driver.current_url,
body=driver.page_source,
encoding='utf-8',
request=request,
)
time.sleep(0.5)
driver.quit()
def driver():
chrome_options = ChromeOptions()
chrome_options.headless = True
driver = webdriver.Chrome(chrome_options=chrome_options,
project_name="CI - Python")
yield driver
driver.quit()
def setup_options(self):
chrome_options = ChromeOptions()
chrome_options.headless = True
flags = {
'h': '--headless',
'u': f'user-agent={UserAgent().random}',
'p': f'--proxy-server={Config.PROXY}',
'w': f'--window-size={Config.WINDOW_SIZE}',
's': '--no-sandbox',
'g': '--disable-gpu',
'i': '--incognito',
't': f'--num-raster-threads={Config.THREADS}',
'#': '--blink-settings=syncXHRInDocumentsEnabled=false',
'@': '--blink-settings=scriptEnabled=false',
'%': '--blink-settings=imagesEnabled=false',
'$': '--blink-settings=pluginsEnabled=false',
'c': '--blink-settings=cookieEnabled=false'
}
if self.options:
for option in self.options:
chrome_options.add_argument(flags.get(option, None))
return chrome_options
def _init_chrome(self, adblock: bool = False, incognito: bool = False) -> Chrome:
empty_dir(str(Path.cwd() / 'chrome-data'))
chr_opt = ChromeOptions()
chr_opt.headless = self.headless
chr_opt.add_argument("--no-sandbox")
chr_opt.add_argument('log-level=2')
chr_opt.add_argument('--disable-logging')
chr_opt.add_argument('--disable-remote-fonts')
chr_opt.add_argument("--user-data-dir=chrome-data")
chr_opt.add_argument("--remote-debugging-port=9222")
chr_opt.add_argument("--disable-infobars")
chr_opt.add_argument("--disable-dev-shm-usage")
chr_opt.add_argument("--ignore-certificate-errors")
chr_opt.add_argument("--disable-gpu")
if adblock and not self.headless: # headless + extensions = crash
chr_opt.add_extension(
r'C:\Users\Roel\PycharmProjects\scrapers\tripadvisor\driver\ublock.crx.crx')
if incognito:
chr_opt.add_argument('--incognito')
if 'posix' in os.name:
chr_opt.binary_location = '/home/vries274/scrapers/tripadvisor/chrome/chrome-linux/chrome'
chrome = Chrome(executable_path=self.CHR_PATH, options=chr_opt)
chrome.set_window_size(1920, 1080)
print('\n --- Browser started --- \n')
return chrome
def driver():
chrome_options = ChromeOptions()
chrome_options.headless = True
chrome_options.set_capability("cloud:URL", os.getenv("TP_CLOUD_URL"))
driver = webdriver.Chrome(chrome_options=chrome_options, project_name="CI - Python")
yield driver
driver.quit()
def create_chrome_driver():
options = ChromeOptions()
options.add_argument("--window-size=1920,1080")
options.add_argument("--start-maximized")
options.headless = headless
return webdriver.Chrome(
executable_path=ChromeDriverManager().install(), options=options)
def choice_browser(request):
cli_input = request.config.getoption("--browser")
if cli_input == "chrome":
options = ChromeOptions()
options.headless = True
wd = webdriver.Chrome(options=options)
request.addfinalizer(wd.quit)
return wd
elif cli_input == "firefox":
options = FirefoxOptions()
options.headless = True
wd = webdriver.Firefox(options=options)
wd.wait = WebDriverWait(wd, 8)
request.addfinalizer(wd.quit)
return wd
def open_the_browser(self):
options = ChromeOptions()
options.headless = False
driver = Chrome(options=options,
executable_path=ChromeDriverManager().install())
sleep(2)
return driver
def __init_options(self):
options = ChromeOptions()
options.experimental_options["useAutomationExtension"] = False
options.add_experimental_option('excludeSwitches', ['enable-logging'])
options.headless = True
# https://stackoverflow.com/questions/43571119/loading-of-unpacked-extensions-is-disabled-by-the-administrator
return options
def main():
"""
メインの処理。
"""
options = ChromeOptions()
# ヘッドレスモードを無効にするには、次の行をコメントアウトする。
options.headless = True
driver = Chrome(options=options) # ChromeのWebDriverオブジェクトを作成する。
# Windows上の仮想マシンの場合は、前の行をコメントアウトして、次の行のコメントアウトを解除する。
# driver = Remote('http://10.0.2.2:4444', options=options)
navigate(driver) # noteのトップページに遷移する。
contents = scrape_contents(driver) # コンテンツのリストを取得する。
logging.info(f'Found {len(contents)} contents.') # 取得したコンテンツの数を表示する。
# スキの数が最も多いコンテンツを取得する。
content = sorted(contents, key=lambda c: c['like'], reverse=True)[0]
logging.info(f'Notifying to Slack: {content["url"]}')
# 取得したコンテンツのスキの数とURLをSlackに通知する。
requests.post(SLACK_INCOMING_WEBHOOK_URL, json={
'text': f':heart: {content["like"]} {content["url"]}',
'unfurl_links': True, # リンクのタイトルや画像を表示する。
})
driver.quit() # ブラウザーを終了する。
def browser():
options = ChromeOptions()
options.headless = True
wd = webdriver.Chrome(options=options)
yield
#request.addfinalizer(wd.quit)
wd.quit()
def _create_driver(self):
browsers = webdriver.DRIVER_PREFERENCE[platform.system()]
for browser in browsers:
kwargs = {}
if self._headless:
from selenium.webdriver import ChromeOptions
chrome_options = ChromeOptions()
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument("--window-size=1280,1024")
chrome_options.add_argument("--disable-dev-shm-usage")
chrome_options.headless = True
kwargs["options"] = chrome_options
for download in [False, True]:
executable = webdriver.executable(browser, download=download)
try:
if executable:
driver = webdriver.start(browser,
executable_path=executable,
**kwargs)
else:
driver = webdriver.start(browser, **kwargs)
return driver, browser
except WebDriverException:
if self.logger.level >= logging.DEBUG:
self.logger.exception(
"Error trying to start with executable: %s",
executable)
raise ValueError("No valid browser found")
def _process_chrome_options(self, opts):
if isinstance(opts, ChromeOptions):
options = opts
else:
options = ChromeOptions()
if 'args' in opts:
for arg in opts.pop('args'):
options.add_argument(arg)
if opts.pop('headless', None) or self.options.pop(
'headless', None):
options.headless = True
if 'binary' in opts or 'binary_location' in opts:
options.binary_location = opts.pop('binary') or opts.pop(
'binary_location')
if 'debugger_address' in opts:
options.debugger_address = opts.pop('debugger_address')
if 'extensions' in opts:
for path in opts.pop('extensions'):
options.add_extension(path)
if 'encoded_extensions' in opts:
for string in opts.pop('encoded_extensions'):
options.add_encoded_extension(string)
if 'experimental_options' in opts:
for name, value in opts.pop('experimental_options').items():
options.add_experimental_option(name, value)
self.selenium_opts['options'] = options
def driver_factory(browser):
if browser == "chrome":
options = ChromeOptions()
options.headless = True
options.add_argument('--ignore-ssl-errors=yes')
options.add_argument('--ignore-certificate-errors')
driver = webdriver.Chrome(options=options)
elif browser == "firefox":
profile = FirefoxProfile()
profile.accept_untrusted_certs = True
options = FirefoxOptions()
options.headless = True
driver = webdriver.Firefox(options=options, firefox_profile=profile)
else:
raise Exception("Driver not supported")
return driver
def get_chrome_options(config):
options = ChromeOptions()
options.headless = config["headless"]
if options.headless:
options.add_argument('--disable-gpu')
options.add_argument('window-size=1920x1080')
return options
def open_the_browser(self):
options = ChromeOptions()
options.headless = False # -> FALSE IF YOU WANT TO SEE THE BROWSER BROWSING
driver = Chrome(options=options, executable_path=ChromeDriverManager().install())
driver.get(self.WEB_LINKS["football"])
sleep(3)
driver.find_element_by_css_selector('#close-cc-bar').click()
return driver
def create_driver(self):
options = ChromeOptions()
options.headless = True
options.add_argument("window-size=1920x1080")
options.add_argument('--no-sandbox')
options.add_argument('--disable-gpu')
return Chrome(options=options)
def open_the_browser(self, link):
# OPEN THE BROWSER
options = ChromeOptions()
options.headless = True # IF YOU WANT TO SEE THE BROWSER -> FALSE
driver = Chrome(options=options, executable_path=ChromeDriverManager().install())
driver.get(self.WEB_LINKS[link])
sleep(4)
return driver
def __init__(self):
# 在初始化淘宝对象时，创建driver
super(Art3spiderSpider, self).__init__(name='art3spider')
option = ChromeOptions()
option.headless = True
self.driver = webdriver.Chrome(
r"/Users/krystal/pythonWokespace/chromedriver",
chrome_options=option)
def __init__(self, headless=True, options=[], path='myengine\chromedriver'):
browser_options = ChromeOptions()
for _ in options:
browser_options.add_argument(_)
browser_options.headless = headless
browser_options.add_experimental_option('excludeSwitches', ['enable-logging'])
Chrome.__init__(self, options=browser_options, executable_path=path)
Browser.__init__(self)
def __init__(self):
options = ChromeOptions()
options.headless = True
options.add_argument('lang=pt-br')
self.driver = Chrome(executable_path='./driver/chromedriver')
self.driver.maximize_window()
self.driver.get('https://web.whatsapp.com/')
sleep(30)
def browser(request):
test_name = request.node.name
logger = logging.getLogger('browser fixture')
logger.info("\nStarted tests {}".format(test_name))
# selenoid = request.config.getoption("--selenoid")
# exec_url = f"http://{selenoid}:4444/wd/hub"
browser = request.config.getoption("--browser")
# capabilities for selenoid
# caps = {"browserName": browser,
# "version": "81.0",
# "enableLog": True,
# "enableVNC": True,
# "screenResolution": "1280x720",
# "name": request.node.name
# }
#
# wd = webdriver.Remote(command_executor=exec_url, desired_capabilities=caps)
# logger.info("\n {} started {}".format(browser, wd.desired_capabilities))
# logger.info(f"\n Start session {wd.session_id}")
# request.addfinalizer(wd.quit)
# return wd
if browser == "chrome":
options = ChromeOptions()
options.headless = True
options.add_argument('--disable-infobars')
options.add_argument('--disable-notifications')
options.add_argument('--disable-web-security')
options.add_argument('--ignore-certificate-errors')
wd = webdriver.Chrome(options=options)
logger.info("\n {} started {}".format(browser,
wd.desired_capabilities))
request.addfinalizer(wd.quit)
return wd
elif browser == "firefox":
options = FirefoxOptions()
options.headless = True
wd = webdriver.Firefox(options=options)
logger.info("\n {} started {}".format(browser,
wd.desired_capabilities))
request.addfinalizer(wd.quit)
return wd
def driver(self):
try:
_ = self._driver.current_window_handle
except:
rpa_logger.info('获取Driver...')
options = ChromeOptions()
options.headless = True
self._driver = Chrome(options=options)
return self._driver
def get_chrome_options(config):
options = ChromeOptions()
options.headless = config["headless"]
options.add_argument("--no-sandbox")
options.add_argument("--disable-dev-shm-usage")
options.add_argument("--disable-notifications")
options.add_argument('--ignore-certificate-errors')
options.add_argument('--allow-running-insecure-content')
return options
def __init__(self, head: bool = False) -> None:
options = ChromeOptions()
options.headless = True
options.add_argument("window-size=1920x1080")
options.add_argument('--no-sandbox')
options.add_argument('--disable-gpu')
# options.add_argument('--disable-dev-shm-usage') # Not used
self.driver = Chrome(options=options)
def browser(): # request
chrome_options = ChromeOptions()
chrome_options.headless = True
# if pytest.getOptinons("browser") == chrome:
wd = webdriver.Chrome(options=chrome_options)
# elif pytest.getOptinons("browser") == firefox:
# wd = webdriver.Firefox(options=firefox_options)
yield
# request.addfinalizer(wd.quit)
wd.quit()
def open_the_browsers(self):
# OPEN THE WEBSITE AND WORK WITH IT
options = ChromeOptions()
options.headless = True # IF YOU WANT TO SEE THE BROWSER -> FALSE
driver = Chrome(options=options, executable_path=ChromeDriverManager().install())
driver_tomorrow = Chrome(options=options, executable_path=ChromeDriverManager().install())
driver.get(self.WEB_LINKS['football_today'])
driver_tomorrow.get(self.WEB_LINKS['football_tomorrow'])
sleep(3)
return driver, driver_tomorrow
