def __set_webdriver_log_level(self, log_level): # Nicer method to setup webdriver's log level (too verbose by default) from selenium.webdriver.remote.remote_connection import LOGGER if log_level: LOGGER.setLevel(log_level) else: LOGGER.setLevel(logging.INFO)
def setUpClass(cls): if settings.GUIDISPLAY == False: settings.DISPLAY.start() LOGGER.setLevel(logging.WARNING) # Instantiating the WebDriver will load your browser cls.wd = CustomWebDriver() cls.server_url = settings.TEST_SERVER_URL cls.base64String = base64.encodestring("%s:%s" % ("ionadmin", "ionadmin")).replace("\n", "") ##cls.server_url = 'http://ts-sandbox.itw' cls.wd.get("%s%s" % (cls.server_url, '/login')) ##the login now persists between tests ##so we only need to login with the username/password if the page ##has the username fields try: #enter the username and password cls.wd.find_css('#id_username').send_keys('ionadmin') cls.wd.find_css("#id_password").send_keys('ionadmin') #click the login link cls.wd.find_element_by_xpath('//button[@type="submit"]').click() #wait for the Ajax on the HOME page cls.wd.wait_for_ajax() except NoSuchElementException: pass
def setUpPackage(): test_app = App() thread = threading.Thread(target=test_app.start) thread.daemon = True thread.start() web_actors['server'] = test_app web_actors['browser'] = Browser() LOGGER.setLevel(logging.WARNING)
def __init__(self, thread_id, config, url): logging.debug("Instantiating the thread '%s' for the class '%s'" % (thread_id, self.__class__.__name__)) self._cfg = config self._url = url self._thread_id = thread_id self.data = None self.log = None self.browser = webdriver.Firefox() LOGGER.setLevel(logging.WARNING) threading.Thread.__init__(self)
def _init_selenium_driver(self): LOGGER.setLevel(logging.WARNING) # CHROMEDRIVER_BIN = '/usr/local/bin/chromedriver' # os.environ['webdriver.chrome.driver'] = CHROMEDRIVER_BIN # self.d = webdriver.Chrome(CHROMEDRIVER_BIN) self.d = webdriver.Firefox() # self.d.implicitly_wait(1) # self.d = webdriver.PhantomJS('/usr/local/bin/phantomjs') self.d.set_window_size(config.BROWSER_WIDTH, config.BROWSER_HEIGHT)
def with_firefox(): LOGGER.setLevel(logging.WARNING) user_platform = platform.platform() if "win" in user_platform: driver_path = os.path.dirname(os.path.realpath(__file__)).replace("app", "conf\geckodriver.exe") elif "linux" in user_platform: driver_path = os.path.dirname(os.path.realpath(__file__)).replace("app", "conf\geckodriver-linux") elif "darwin" in user_platform: driver_path = os.path.dirname(os.path.realpath(__file__)).replace("app", "conf\geckodriver-macos") world.browser = webdriver.Firefox(executable_path = driver_path) world.browser.maximize_window() yield world.browser.quit() delattr(world, 'browser')
def __init__(self, xvfb=True, driver='Firefox', mootools=False, timeout=90, width=1440, height=1200, firefox_version=46, desired_capabilities='FIREFOX', command_executor='http://127.0.0.1:4444/wd/hub'): self.driver = driver # Firefox, PhantomJS (Must be installed...) self.xvfb = xvfb # This is for headless running. self.mootools = mootools # Use MooTools instead of jQuery self.timeout = timeout # Global timeout option for all wait_* functions self.width = width # XVFB virtual monitor width self.height = height # XVFB virtual monitor width self.desired_capabilities = desired_capabilities self.command_executor = command_executor if os.environ.get('skip_xvfb'): self.xvfb = False # Turn off annoying selenium logs s_logger.setLevel(logging.WARNING)
def __init__(self): path = abspath(join(dirname(dirname(__file__)), "phantomjs-2.1.1-64")) sys.path.append(path) # phantomjs needs to be in path when running from pycharm cap = dict(DesiredCapabilities.PHANTOMJS) cap["phantomjs.page.settings.userAgent"] = ("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0 ") service_args=["--webdriver-loglevel=DEBUG", "--cookies-file=ghostdriver.cookies"] ghostdriver_logger.setLevel(logging.DEBUG) #self.driver = webdriver.Firefox() self.driver = webdriver.PhantomJS(executable_path=path, desired_capabilities=cap, service_args=service_args) self.driver.timeout = { # adds field to use only one of these values for a timeout "implicit": 10, "explicit": 10, "page_load": 30 } self.driver.implicitly_wait(self.driver.timeout["implicit"]) self.driver.set_window_size(1280, 768) self.driver.maximize_window() self.driver.set_page_load_timeout(self.driver.timeout["page_load"]) # driver.get uses this timeout when calling requests.get
def _init_selenium_driver(self): LOGGER.setLevel(logging.WARNING) CHROMEDRIVER_BIN = "/usr/lib/chromium-browser/chromedriver" # os.environ['webdriver.chrome.driver'] = CHROMEDRIVER_BIN self.d = webdriver.Chrome(executable_path=CHROMEDRIVER_BIN) # firefox_profile = webdriver.FirefoxProfile() # firefox_profile.set_preference("network.proxy.type", 1) # firefox_profile.set_preference("network.proxy.http", '127.0.0.1') #set your ip # firefox_profile.set_preference("network.proxy.http_port", 8080) #set your port # self.d = webdriver.Firefox(firefox_profile=firefox_profile) # self.d = webdriver.Firefox() # self.d.implicitly_wait(1) # self.d = webdriver.PhantomJS('/home/tunnelshade/Downloads/phantomjs') # service_args=[ # '--proxy=127.0.0.1:8080', # '--proxy-tpe=http']) self.d.set_window_size(config.BROWSER_WIDTH, config.BROWSER_HEIGHT)
from selenium.webdriver.chrome.options import Options from selenium.webdriver.common.action_chains import ActionChains from selenium.webdriver.common.by import By from selenium.webdriver.support import expected_conditions as ec from selenium.webdriver.support.wait import WebDriverWait from selenium.webdriver.remote.remote_connection import LOGGER as selenium_logger from urllib3.connectionpool import log as urllib_logger from logging import WARNING as logging_WARNING options = Options() options.add_experimental_option("excludeSwitches", ["enable-automation", "enable-logging"]) options.add_experimental_option("useAutomationExtension", False) # CHROME ONLY option to prevent Restore Session popup options.add_argument("--disable-session-crashed-bubble") selenium_logger.setLevel(logging_WARNING) urllib_logger.setLevel(logging_WARNING) class AnyEc: """Use with WebDriverWait to combine expected_conditions in an OR. """ def __init__(self, *args): self.ecs = args def __call__(self, driver): for fn in self.ecs: try: if fn(driver): return True
from splinter import Browser import datetime import json import os import logging import time logging.basicConfig() logger = logging.getLogger("splinter_tests.__init__") # Quiet down the Selenium logging from selenium.webdriver.remote.remote_connection import \ LOGGER as _selenium_logger _selenium_logger.setLevel(logging.INFO) SERVER_URL = "http://localhost:5859" #SERVER_URL = "https://xbeewifi.herokuapp.com" #SERVER_URL = "https://xbgw-333.herokuapp.com" def make_url(uri): return '{}{}'.format(SERVER_URL, uri) browser = None driver = "phantomjs" def _put_local_phantomjs_on_path(): # Use local PhantomJS installation (from node_modules) by putting the path
def run(url): global wb, ws, driver # Init start time start = datetime.datetime.now().replace(microsecond=0) # Init Log log_file = open(LOG_FILE, 'w+') log_file.close() # Init Driver LOGGER.setLevel(logging.DEBUG) driver = init_driver() if driver is None: sys.exit() # Init Excel wb = openpyxl.Workbook() ws = wb.active # TODO: Check if max page exists # Init Scraping original_url = url proceed = True try: driver.get(original_url) title = driver.title except Exception as e: log("ERROR Failed openning " + original_url + " - " + str(e).split(os.linesep, 1)[0]) close_driver() proceed = False if proceed is True: # Get all URLs ads_urls = [] for page_index in range(1, args.pages + 1): if driver is None: driver = init_driver() if driver is None: log("WARN Failed to initialize driver, page " + page_index + " will be skipped") continue index = original_url.rfind('/') url = original_url[:index] + '/page-' + str( page_index) + original_url[index:] page_urls = get_page_ads(url, page_index) if len(page_urls) == 0: continue for ad_url in page_urls: if ad_url not in ads_urls: ads_urls.append(ad_url) if (page_index) % 10 == 0 or page_index == args.pages: close_driver() # Parse ads total = len(ads_urls) log("INFO * Total of " + str(total) + " ads will be scraped") for index, ad_url in enumerate(ads_urls): if driver is None: driver = init_driver() log("INFO Parsing " + str(index + 1) + "/" + str(total) + ": " + ad_url) try: parse_ad(ad_url) except selenium.common.exceptions.TimeoutException as e: log("ERROR Failed to load page " + str(page_index) + " before time out elapsed, closing driver...") close_driver() except Exception as e: log("ERROR Failed to parse " + ad_url + " - (" + str(e).split(os.linesep, 1)[0] + ")") if (index + 1) % 20 == 0 or index + 1 == total: close_driver() # Save Results to Excel try: if os.path.isfile(RESULTS_FILE): os.remove(RESULTS_FILE) wb.save(RESULTS_FILE) log("INFO Results saved as " + RESULTS_FILE + " in " + DIR_PATH) except Exception as e: log("ERROR Failed to save results " + str(e)) # Calculate Duration end = datetime.datetime.now().replace(microsecond=0) # Send eMail if args.mail is True and proceed is True: if can_send_email(): from_name = "Kijiji Scrapper" if os.path.isfile(RESULTS_FILE): message = title + os.linesep message += original_url + os.linesep message += "Duration - " + str(end - start) + os.linesep send_mail(from_name, args.recipients, "Kijiji Scraping Results", message, [RESULTS_FILE, LOG_FILE], args.smtp_server, args.smtp_server_port, args.smtp_server_username, args.smtp_server_password, True) else: message = "Operation Failed." + os.linesep message += "Logs Attached" + os.linesep message += "Duration - " + str(end - start) send_mail(from_name, args.recipients, "Kijiji Scraping Results - Failed", "Logs attached", [LOG_FILE], args.smtp_server, args.smtp_server_port, args.smtp_server_username, args.smtp_server_password, True) log("INFO Sent results by mail to: " + ','.join(args.recipients)) else: log("ERROR Can not send eMail") log("INFO Script finsihed in " + str(end - start)) # Cleanup if (args.mail): if os.path.exists(RESULTS_FILE): os.remove(RESULTS_FILE) if os.path.exists(LOG_FILE): os.remove(LOG_FILE) print("Bye.")
import logging import pytest from selenium.webdriver.remote.remote_connection import LOGGER LOGGER.setLevel(logging.WARNING) def test_checkbox_set_to_true(test_page): test_page.navigate() test_page.checkbox.set_to(True) assert test_page.checkbox.checked def test_checkbox_set_to_false(test_page): test_page.navigate() test_page.checkbox.check() assert test_page.checkbox.checked test_page.checkbox.set_to(False) assert test_page.checkbox.checked is False def test_checkbox_toggle_on(test_page): test_page.navigate() test_page.checkbox.toggle()
def download(cfg): from selenium import webdriver from selenium.webdriver.common.keys import Keys from selenium.webdriver.remote.remote_connection import LOGGER LOGGER.setLevel(logging.WARNING) retCode = False filename_new = cfg.get('download', 'filename_new') filename_old = cfg.get('download', 'filename_old') filename_in = cfg.get('basic', 'filename_in') # login = cfg.get('download','login' ) # password = cfg.get('download','password' ) url_base = cfg.get('download', 'url_base') url_file = cfg.get('download', 'url_file') download_path = os.path.join(os.getcwd(), 'tmp') if not os.path.exists(download_path): os.mkdir(download_path) for fName in os.listdir(download_path): os.remove(os.path.join(download_path, fName)) dir_befo_download = set(os.listdir(download_path)) if os.path.exists('geckodriver.log'): os.remove('geckodriver.log') try: ffprofile = webdriver.FirefoxProfile() ffprofile.set_preference("browser.download.dir", download_path) ffprofile.set_preference("browser.download.folderList", 2) ffprofile.set_preference( "browser.helperApps.neverAsk.saveToDisk", ",application/octet-stream" + ",application/vnd.ms-excel" + ",application/vnd.msexcel" + ",application/x-excel" + ",application/x-msexcel" + ",application/zip" + ",application/xls" + ",application/vnd.ms-excel" + ",application/vnd.ms-excel.addin.macroenabled.12" + ",application/vnd.ms-excel.sheet.macroenabled.12" + ",application/vnd.ms-excel.template.macroenabled.12" + ",application/vnd.ms-excelsheet.binary.macroenabled.12" + ",application/vnd.ms-fontobject" + ",application/vnd.ms-htmlhelp" + ",application/vnd.ms-ims" + ",application/vnd.ms-lrm" + ",application/vnd.ms-officetheme" + ",application/vnd.ms-pki.seccat" + ",application/vnd.ms-pki.stl" + ",application/vnd.ms-word.document.macroenabled.12" + ",application/vnd.ms-word.template.macroenabed.12" + ",application/vnd.ms-works" + ",application/vnd.ms-wpl" + ",application/vnd.ms-xpsdocument" + ",application/vnd.openofficeorg.extension" + ",application/vnd.openxmformats-officedocument.wordprocessingml.document" + ",application/vnd.openxmlformats-officedocument.presentationml.presentation" + ",application/vnd.openxmlformats-officedocument.presentationml.slide" + ",application/vnd.openxmlformats-officedocument.presentationml.slideshw" + ",application/vnd.openxmlformats-officedocument.presentationml.template" + ",application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + ",application/vnd.openxmlformats-officedocument.spreadsheetml.template" + ",application/vnd.openxmlformats-officedocument.wordprocessingml.template" + ",application/x-ms-application" + ",application/x-ms-wmd" + ",application/x-ms-wmz" + ",application/x-ms-xbap" + ",application/x-msaccess" + ",application/x-msbinder" + ",application/x-mscardfile" + ",application/x-msclip" + ",application/x-msdownload" + ",application/x-msmediaview" + ",application/x-msmetafile" + ",application/x-mspublisher" + ",application/x-msschedule" + ",application/x-msterminal" + ",application/x-mswrite" + ",application/xml" + ",application/xml-dtd" + ",application/xop+xml" + ",application/xslt+xml" + ",application/xspf+xml" + ",application/xv+xml" + ",application/excel") if os.name == 'posix': # driver = webdriver.Firefox(ffprofile, executable_path=r'/usr/local/Cellar/geckodriver/0.19.1/bin/geckodriver') driver = webdriver.Firefox( ffprofile, executable_path=r'/usr/local/bin/geckodriver') elif os.name == 'nt': driver = webdriver.Firefox(ffprofile) driver.implicitly_wait(10) driver.set_page_load_timeout(10) driver.get(url_base) try: driver.get(url_file) except Exception as e: log.debug('Exception: <' + str(e) + '>') except Exception as e: log.debug('Exception: <' + str(e) + '>') driver.quit() dir_afte_download = set(os.listdir(download_path)) new_files = list(dir_afte_download.difference(dir_befo_download)) print(new_files) if len(new_files) == 0: log.error('Не удалось скачать файл прайса ') retCode = False elif len(new_files) > 1: log.error('Скачалось несколько файлов. Надо разбираться ...') retCode = False else: new_file = new_files[0] # загружен ровно один файл. new_ext = os.path.splitext(new_file)[-1].lower() DnewFile = os.path.join(download_path, new_file) new_file_date = os.path.getmtime(DnewFile) log.info( 'Скачанный файл ' + new_file + ' имеет дату ' + time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(new_file_date))) print(new_ext) if new_ext in ('.xls', '.xlsx', '.xlsb', '.xlsm', '.csv'): if os.path.exists(filename_new) and os.path.exists(filename_old): os.remove(filename_old) os.rename(filename_new, filename_old) if os.path.exists(filename_new): os.rename(filename_new, filename_old) shutil.copy2(DnewFile, filename_new) retCode = True elif new_ext == '.zip': # ветка устаревшая, не проверялась # Архив. Обработка не завершена log.debug('Zip-архив. Разархивируем.') work_dir = os.getcwd() os.chdir(os.path.join(download_path)) dir_befo_download = set(os.listdir(os.getcwd())) os.system('unzip -oj ' + new_file) os.remove(new_file) dir_afte_download = set(os.listdir(os.getcwd())) new_files = list(dir_afte_download.difference(dir_befo_download)) os.chdir(work_dir) if len(new_files) == 1: new_file = new_files[0] # разархивирован ровно один файл. new_ext = os.path.splitext(new_file)[-1] DnewFile = os.path.join(download_path, new_file) new_file_date = os.path.getmtime(DnewFile) log.debug('Файл из архива ' + DnewFile + ' имеет дату ' + time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(new_file_date))) if os.path.exists(filename_in) and os.path.exists( 'filename_old'): os.remove('filename_old') os.rename(filename_in, 'filename_old') if os.path.exists(filename_in): os.rename(filename_new, 'filename_old') shutil.copy2(DnewFile, filename_in) retCode = True elif len(new_files) > 1: log.debug('В архиве не единственный файл. Надо разбираться.') retCode = False else: log.debug( 'Нет новых файлов после разархивации. Загляни в папку юниттеста поставщика.' ) retCode = False return retCode
import os, inspect, nose, re, logging DB_URL_RE = re.compile(r'([^\:]+)\:\/\/([^\:]+)\:([^\@]+)\@([^\/]+)\/([^\s]+)') environ = {} if not pylons.test.pylonsapp: logging.error("Cannot run tests: need to specify config file") # if this isn't set up right, let it fail in order for nose to display the error SetupCommand('setup-app').run([pylons.test.pylonsapp.config['__file__']]) try: from selenium.webdriver.remote.remote_connection import LOGGER LOGGER.setLevel(logging.INFO) except ImportError, e: print "These tests are likely going to fail as Selenium had an import error..." def CSS(source, selector): return source.find_elements_by_css_selector(selector) class SeleniumContextTest(TestCase): """ A Selenium test that correctly operates within a QTools context. Whether or not a subclass of this test runs depends on whether it is in a directory specified by the 'nose.selenium.test_paths' variable in the active Pylons configuration. It will also trigger Firefox as the webdriver to check the front-end
def main(ctx, driver, profile, headless, delay, variance, wait, testrun, sound): """A bot that purchases items online, rendering them bought. By default, bought uses the parameters specified in config.ini for configuration but these can also be specified via CLI. Made with <3 by jsonV """ ctx.ensure_object(dict) ctx.obj["main"] = ctx.params drive = None fp = None no_head = False eager = False invoke_newegg = False wait = 0 logging.basicConfig( filemode="a", filename="bought.log", format="[%(asctime)s] [%(levelname)s] %(message)s", ) log = logging.getLogger("bought") log.setLevel(logging.DEBUG) selenium_logger.setLevel(logging.WARNING) urllibLogger.setLevel(logging.WARNING) sh = logging.StreamHandler() sh.setLevel(logging.DEBUG) formatter = logging.Formatter("[%(asctime)s] [%(levelname)s] %(message)s", "%Y-%m-%d %H:%M:%S") sh.setFormatter(formatter) log.addHandler(sh) latest, current = check() if latest > current: log.warn( f"You are running bought v{current}. Please upgrade to v{latest} with: 'pip install --upgrade bought'" ) # Override defaults if config.ini variable specified if ctx.config: driver = ctx.config["Main"]["Driver"].lower() if ctx.config["Main"]["Headless"].lower() == "true": no_head = True if ctx.config["Newegg"]["Enabled"].lower() == "true": items = ctx.config["Newegg"]["Items"] invoke_newegg = True if ctx.config["Main"]["ImplicitWait"]: wait = int(ctx.config["Main"]["ImplicitWait"]) # Override if CLI argument specified if driver: drive = driver if headless: no_head = headless if drive == "firefox": log.info(f"Using FireFox webdriver.") if ctx.config: if os.path.isdir(ctx.config["Main"]["BrowserProfile"]): log.info( f'Logging profile: {ctx.config["Main"]["BrowserProfile"]}') fp = webdriver.FirefoxProfile( ctx.config["Main"]["BrowserProfile"]) log.info("Finished loading profile.") else: log.info("Using default blank profile.") options = webdriver.FirefoxOptions() elif drive == "chrome" or drive == "chromium": log.info(f"Using Chrome webdriver.") options = webdriver.ChromeOptions() if ctx.config: if ctx.config["Main"]["BrowserProfile"]: log.info( f'Logging profile: {ctx.config["Main"]["BrowserProfile"]}') options.add_argument( f'--user-data-dir={ctx.config["Main"]["BrowserProfile"]}') else: log.info("Using default blank profile.") if no_head: options.set_headless() if eager: options.page_load_strategy = "eager" if drive == "firefox": log.info("Starting webdriver.") driver = webdriver.Firefox(fp, firefox_options=options) log.info("Successfully created webdriver.") elif drive == "chrome" or drive == "chromium": log.info("Starting webdriver.") driver = webdriver.Chrome(chrome_options=options) log.info("Successfully created webdriver.") else: log.warn("No webdriver specified via config.ini or CLI!") log.warn( "Ensure config.ini specifies [Main][Driver] or CLI specifies driver via `bought -d firefox|chrome|chromium!" ) if wait: log.info("Setting implicit wait time.") driver.implicitly_wait(wait) if drive: ctx.obj["driver"] = driver if invoke_newegg: log.info("Starting Newegg script.") ctx.obj["config"] = ctx.config ctx.invoke(newegg, items=items)
from selenium.webdriver.support.ui import WebDriverWait from anime_downloader.const import get_random_header from selenium.webdriver.common.by import By from urllib.parse import urlencode from urllib.parse import urlsplit from selenium import webdriver from bs4 import BeautifulSoup from logging import exception from sys import platform import requests import os import logging import click import time import json serverLogger.setLevel(logging.ERROR) logger = logging.getLogger(__name__) def get_data_dir(): ''' Gets the folder directory selescrape will store data, such as cookies or browser extensions and logs. ''' APP_NAME = 'anime downloader' return os.path.join(click.get_app_dir(APP_NAME), 'data') def open_config(): from anime_downloader.config import Config return Config
import unittest import yaml import time import os import roles from browser import Browser from ppm_fhir.fhir_services import FHIRServices import logging from selenium.webdriver.remote.remote_connection import LOGGER as SeleniumLogger SeleniumLogger.setLevel(logging.WARNING) logger = logging.getLogger('ppm-test-suite') class PPMTestCase(unittest.TestCase): @classmethod def setUpClass(cls): # Set the environment cls.environment = os.environ.get('PPM_TEST_ENVIRONMENT', 'local') cls.test_integrations = os.environ.get('PPM_TEST_INTEGRATIONS', False) # Get the path path = os.path.dirname(__file__) # Determine places with open(os.path.join(path, 'urls.yml'), 'r') as stream: try:
# import all required modules from selenium import webdriver import time, re, hashlib, csv import pandas as pd from selenium.webdriver.common.keys import Keys import usaddress import logging, datetime, sys from selenium.webdriver.remote.remote_connection import LOGGER LOGGER.setLevel(logging.ERROR) username = '******' salutation = '' #Mr. Miss client_name = '' client_address = '' billing_card_number = '' card_cvv_number = '' billing_address = '' billing_expiry_date = '' # log file initialize logging.basicConfig( level=logging.DEBUG, filename='logfile_' + username + '.log', # log to this file format='%(asctime)s -- %(message)s') # include timestamp headers = ['Date' , 'Client_Name', 'State', 'County', 'searchterm', 'recordyear', 'last_document_type', \ 'last_document_left', 'right_side_name', 'right_side_address', 'right_side_date', 'Doctypes', 'Record_Count', 'last data record found', \ 'parsed_rows', 'client_name', 'client_address', 'billing_card_number', 'card_cvv_number', 'billing_address', 'billing_expiry_date'] ignore_keywords = [ 'escrow', 'esq', 'inc', 'law', 'service', 'law', 'Timeshare', 'associates', 'recorder', 'recorders', '$', 'iiii', 'service', 'closing', 'title'
def __init__(self, id): self.endpoint_html = dict() self.endpoint_soup = dict() self.id = id LOGGER.setLevel(logging.WARNING) # remove Selenium logging
import logging import os # from pyvirtualdisplay import Display from selenium import webdriver from selenium.webdriver.common.keys import Keys from selenium.webdriver.remote.remote_connection import LOGGER from subprocess import call import time import unittest from webdriver_utils import ( Server, Client ) LOGGER.setLevel(logging.WARNING) # silence logging """ this setting controls whether the browser will be visible (1) or not (0) """ is_visible = 1 # configuration of testing framework cfg = { 'app': { 'host': '0.0.0.0', 'port': '6543', 'db': 'webdrivertest.db', 'ini': "webdrivertest.ini", 'appSettings': {}, },
def _instantiate_chromedriver(ctx: Context) -> None: """Attempt to start the chromedriver, retrying if there are connection errors. Args: ctx: The behave context object. Raises: :py:class:`.ConnectionResetError`: If starting chromedriver fails too many times. """ # Set the selenium logs to warning only LOG.setLevel(logging.WARNING) chrome_options = webdriver.ChromeOptions() # Prevent images from loading (should decrease load times) prefs = {"profile.managed_default_content_settings.images": 1} chrome_options.add_experimental_option("prefs", prefs) chrome_options.add_argument("--no-sandbox") # Ignore CORS errors chrome_options.add_argument("--disable-web-security") # set logging capability chrome_options.set_capability("loggingPrefs", {"browser": "ALL"}) # if ctx.zap_proxy_url: # chrome_options.add_argument(f"--proxy-server={ctx.zap_proxy_url}") chrome_options.add_argument("--window-size=1920,1080") # Attempt the connection... Max attempts 3 attempts_remaining = 3 while attempts_remaining > 0: try: LOGGER.info("Instantiating chromedriver...") # use the chromedriver binary loader. this forces the location on path chromedriver_binary.add_chromedriver_to_path() ctx.driver = webdriver.Chrome(chrome_options=chrome_options) LOGGER.debug( f"Chromedriver running from {chromedriver_binary.chromedriver_filename}" ) LOGGER.info("Connected to chromedriver successfully!") break except (ConnectionResetError, ProtocolError): # one attempt used... attempts_remaining -= 1 LOGGER.warning( "Connection was refused, will try again {} more " "time{}".format( attempts_remaining, "" if attempts_remaining == 1 else "s" ) ) # sleep 3 seconds between attempts time.sleep(3) else: raise ConnectionResetError( "Failed connecting to chromedriver after exhausting all 3 attempts. Giving up!" ) # Set the latency for the browser if not defaulted to 0 latency = ctx.latency if latency != "None": LOGGER.debug(f"Non default latency was detected as: {latency}") _set_browser_latency(ctx, latency)
patch_pytest_selenium_screenshot() def pytest_addoption(parser): """Add support for headless option to pytest-selenium.""" parser.addoption('--headless', action='store', help='enable headless mode for supported browsers.') # Disable selenium's remote_connection.py DEBUG statements that pollute # test stdout. # They aren't needed as driver logs are included into links of HTML report. SELENIUM_LOGGER.setLevel(logging.INFO) @pytest.fixture(autouse=True) def reset_dict_executed_fixtures(): """Reset dict_executed_fixtures between tests""" dynamic_fixtures.dict_executed_fixtures = {} @pytest.fixture(scope="session") def session_capabilities(session_capabilities): """Log browser (console log) and performance (request / response headers) data. They will appear in `Links` section of pytest-html's html report. """ session_capabilities["loggingPrefs"] = {
def process_request(self, request, spider): # Called for each request that goes through the downloader # middleware. # Must either: # - return None: continue processing this request # - or return a Response object # - or return a Request object # - or raise IgnoreRequest: process_exception() methods of # installed downloader middleware will be called if (spider.name != 'song_chasing_spidy' or 'robots.txt' in request.url): return None #Init of the selenium webdriver driver = webdriver.Chrome( executable_path=ChromeDriverManager().install()) #Selenium logs set to warning LOGGER.setLevel(logging.WARNING) #Execute request driver.get(request.url) #Wait until links to favorite songs of profile is displayed link_to_favorites = WebDriverWait(driver, 5).until( EC.element_to_be_clickable( (By.XPATH, '//a[contains(@class, "sidebarHeader ")]'))) #Navigate to favorite songs page link_to_favorites.click() # Scroll to bottom of the page # Get scroll height last_height = driver.execute_script( "return document.body.scrollHeight") while True: # Scroll down to bottom driver.execute_script( "window.scrollTo(0, document.body.scrollHeight);") # Wait to load page time.sleep(0.5) # Calculate new scroll height and compare with last scroll height new_height = driver.execute_script( "return document.body.scrollHeight") if new_height == last_height: break last_height = new_height #Set return parameters body = to_bytes(driver.page_source) url = driver.current_url #Close driver driver.close() #Return response to spider return HtmlResponse(url, body=body, encoding='utf-8', request=request)
import os from logging import WARNING from splinter.browser import Browser from selenium.webdriver.remote.remote_connection import LOGGER LOGGER.setLevel(WARNING) def before_all(context): browser = context.config.browser or os.environ.get('browser') or 'firefox' context.browser = Browser(browser) def after_all(context): context.browser.quit() context.browser = None
import os import logging LOG_LEVEL = os.getenv("LOG_LEVEL", 'INFO') logging.basicConfig(level=LOG_LEVEL) logging.getLogger().setLevel(LOG_LEVEL) LOG = logging.getLogger(__name__) # reduce log levels for selenium. debug is chatty, doesnt default to root level from selenium.webdriver.remote.remote_connection import LOGGER as serverLogger serverLogger.setLevel(logging.WARNING) logging.getLogger("yahoo_oauth").setLevel(level=logging.INFO) def print_logger_heirarchy(log): print( f'{log.name}, effective_level: {log.getEffectiveLevel()}, Type: {type(log)}' ) if log.parent: print_logger_heirarchy(log.parent)
from selenium.webdriver.common.keys import Keys from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.common.by import By import urlparse from demerio_utils.log import * from selenium.webdriver.remote.remote_connection import LOGGER LOGGER.setLevel(logging.WARNING) TIME_TO_WAIT = 10 def convert_url_to_query_dict(url): return dict(urlparse.parse_qsl(urlparse.urlsplit(url).query)) def enter_text_by_id(driver, id, content, press_return=True): wait = WebDriverWait(driver, TIME_TO_WAIT) element = wait.until(EC.element_to_be_clickable((By.ID, id))) element.send_keys(content) if press_return: element.send_keys(Keys.RETURN) def click_button_by_id(driver, id): wait = WebDriverWait(driver, TIME_TO_WAIT) btn = wait.until(EC.element_to_be_clickable((By.ID, id))) btn.click() def enter_text_by_xpath(driver, xpath, content, press_return=True):
def set_level_logging(level): urllibLogger.setLevel(level) LOGGER.setLevel(level)
def setUp(cls, agent=False): LOGGER.setLevel(logging.WARNING) cls.driver = cls.create_driver(agent=agent)
from time import sleep from .utils import ( Utils, URLS, TIMEOUTS, Participant, tries_n_time_until_true, names_cleaner, ) from .exceptions import LoginError from .interfaces import IModule, IOnAirModule from selenium.webdriver.remote.remote_connection import LOGGER import logging LOGGER.setLevel(logging.WARNING) # reducing selenium verbosity @retry(stop_max_attempt_number=3) def _create_hangout_event(browser, name, attendees): """ Creates hangout event on google plus. As arguments takes event name and attendees list, also should be provided with 'browser' object where visitor is logged in """ browser.get(URLS.onair) browser.by_text('Create a Hangout On Air').click(TIMEOUTS.fast) # Setting name browser.xpath( '//input[@aria-label="Give it a name"]').send_keys(name) # cleaning 'share' field and send attendees list there
def __init__(self, task_queue, verbose): ''' Initializes one scraper process Args: task_queue: task queue verbose (bool): print INFO or not Returns: Nothing ''' multiprocessing.Process.__init__(self) self.task_queue = task_queue self.verbose = verbose # init actions self.web_actions = json.load(open('Configs/actions.json', 'r')) for i in self.web_actions.keys(): if i[0] is '_': continue tmp_file = import_module('Scraper.actions.' + self.web_actions[i]) self.web_actions[i] = getattr(tmp_file, self.web_actions[i]) # inline actions -> flowchart related self.inline_actions = { 'goto', 'loop', 'auto_get_data' # other inline actions } # init firefox driver LOGGER.setLevel(logging.WARNING) self.options = webdriver.FirefoxProfile() if platform.system() == 'Linux': # extension extensions = ['Scraper/driver/ImageBlock.xpi'] self.driver_path = 'Scraper/driver/geckodriver' # disable Images and Flash self.options.set_preference('permissions.default.image', 2) self.options.set_preference( 'dom.ipc.plugins.enabled.libflashplayer.so', 'false') # add extensions to firefoxx for ext in extensions: self.options.add_extension(ext), self.options.set_preference( 'extensions.ImageBlock.currentVersion', '3.1') else: logger(ERR, 'Platform Not Supported') # run self.browser = webdriver.Firefox(firefox_profile=self.options, executable_path=self.driver_path) self.browser.maximize_window() try: # initiate Global event checker self.gev_checker = event_checker.event_checker( 'Configs/global_events.json', self.browser) if type(self.gev_checker) is str: logger(ERR, self.gev_checker) except: logger(WARN, 'Cannot Create Event Handler') self.gev_checker = None # initiate auto get data try: self.auto_getter = auto_get_data('Configs/data_collector.json', self.browser) except: logger(WARN, 'Cannot Create Auto Collector') self.auto_getter = None