def __set_webdriver_log_level(self, log_level):
# Nicer method to setup webdriver's log level (too verbose by default)
from selenium.webdriver.remote.remote_connection import LOGGER
if log_level:
LOGGER.setLevel(log_level)
else:
LOGGER.setLevel(logging.INFO)
def setUpClass(cls):
if settings.GUIDISPLAY == False:
settings.DISPLAY.start()
LOGGER.setLevel(logging.WARNING)
# Instantiating the WebDriver will load your browser
cls.wd = CustomWebDriver()
cls.server_url = settings.TEST_SERVER_URL
cls.base64String = base64.encodestring("%s:%s" % ("ionadmin", "ionadmin")).replace("\n", "")
##cls.server_url = 'http://ts-sandbox.itw'
cls.wd.get("%s%s" % (cls.server_url, '/login'))
##the login now persists between tests
##so we only need to login with the username/password if the page
##has the username fields
try:
#enter the username and password
cls.wd.find_css('#id_username').send_keys('ionadmin')
cls.wd.find_css("#id_password").send_keys('ionadmin')
#click the login link
cls.wd.find_element_by_xpath('//button[@type="submit"]').click()
#wait for the Ajax on the HOME page
cls.wd.wait_for_ajax()
except NoSuchElementException:
pass
def setUpPackage():
test_app = App()
thread = threading.Thread(target=test_app.start)
thread.daemon = True
thread.start()
web_actors['server'] = test_app
web_actors['browser'] = Browser()
LOGGER.setLevel(logging.WARNING)
def __init__(self, thread_id, config, url):
logging.debug("Instantiating the thread '%s' for the class '%s'" % (thread_id, self.__class__.__name__))
self._cfg = config
self._url = url
self._thread_id = thread_id
self.data = None
self.log = None
self.browser = webdriver.Firefox()
LOGGER.setLevel(logging.WARNING)
threading.Thread.__init__(self)
def _init_selenium_driver(self):
LOGGER.setLevel(logging.WARNING)
# CHROMEDRIVER_BIN = '/usr/local/bin/chromedriver'
# os.environ['webdriver.chrome.driver'] = CHROMEDRIVER_BIN
# self.d = webdriver.Chrome(CHROMEDRIVER_BIN)
self.d = webdriver.Firefox()
# self.d.implicitly_wait(1)
# self.d = webdriver.PhantomJS('/usr/local/bin/phantomjs')
self.d.set_window_size(config.BROWSER_WIDTH, config.BROWSER_HEIGHT)
def with_firefox():
LOGGER.setLevel(logging.WARNING)
user_platform = platform.platform()
if "win" in user_platform:
driver_path = os.path.dirname(os.path.realpath(__file__)).replace("app", "conf\geckodriver.exe")
elif "linux" in user_platform:
driver_path = os.path.dirname(os.path.realpath(__file__)).replace("app", "conf\geckodriver-linux")
elif "darwin" in user_platform:
driver_path = os.path.dirname(os.path.realpath(__file__)).replace("app", "conf\geckodriver-macos")
world.browser = webdriver.Firefox(executable_path = driver_path)
world.browser.maximize_window()
yield
world.browser.quit()
delattr(world, 'browser')
def __init__(self, xvfb=True, driver='Firefox', mootools=False,
timeout=90, width=1440, height=1200, firefox_version=46,
desired_capabilities='FIREFOX',
command_executor='http://127.0.0.1:4444/wd/hub'):
self.driver = driver # Firefox, PhantomJS (Must be installed...)
self.xvfb = xvfb # This is for headless running.
self.mootools = mootools # Use MooTools instead of jQuery
self.timeout = timeout # Global timeout option for all wait_* functions
self.width = width # XVFB virtual monitor width
self.height = height # XVFB virtual monitor width
self.desired_capabilities = desired_capabilities
self.command_executor = command_executor
if os.environ.get('skip_xvfb'):
self.xvfb = False
# Turn off annoying selenium logs
s_logger.setLevel(logging.WARNING)
def __init__(self):
path = abspath(join(dirname(dirname(__file__)), "phantomjs-2.1.1-64"))
sys.path.append(path) # phantomjs needs to be in path when running from pycharm
cap = dict(DesiredCapabilities.PHANTOMJS)
cap["phantomjs.page.settings.userAgent"] = ("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0 ")
service_args=["--webdriver-loglevel=DEBUG", "--cookies-file=ghostdriver.cookies"]
ghostdriver_logger.setLevel(logging.DEBUG)
#self.driver = webdriver.Firefox()
self.driver = webdriver.PhantomJS(executable_path=path, desired_capabilities=cap, service_args=service_args)
self.driver.timeout = { # adds field to use only one of these values for a timeout
"implicit": 10,
"explicit": 10,
"page_load": 30
}
self.driver.implicitly_wait(self.driver.timeout["implicit"])
self.driver.set_window_size(1280, 768)
self.driver.maximize_window()
self.driver.set_page_load_timeout(self.driver.timeout["page_load"]) # driver.get uses this timeout when calling requests.get
def _init_selenium_driver(self):
LOGGER.setLevel(logging.WARNING)
CHROMEDRIVER_BIN = "/usr/lib/chromium-browser/chromedriver"
# os.environ['webdriver.chrome.driver'] = CHROMEDRIVER_BIN
self.d = webdriver.Chrome(executable_path=CHROMEDRIVER_BIN)
# firefox_profile = webdriver.FirefoxProfile()
# firefox_profile.set_preference("network.proxy.type", 1)
# firefox_profile.set_preference("network.proxy.http", '127.0.0.1') #set your ip
# firefox_profile.set_preference("network.proxy.http_port", 8080) #set your port
# self.d = webdriver.Firefox(firefox_profile=firefox_profile)
# self.d = webdriver.Firefox()
# self.d.implicitly_wait(1)
# self.d = webdriver.PhantomJS('/home/tunnelshade/Downloads/phantomjs')
# service_args=[
# '--proxy=127.0.0.1:8080',
# '--proxy-tpe=http'])
self.d.set_window_size(config.BROWSER_WIDTH, config.BROWSER_HEIGHT)
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as ec
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.remote.remote_connection import LOGGER as selenium_logger
from urllib3.connectionpool import log as urllib_logger
from logging import WARNING as logging_WARNING
options = Options()
options.add_experimental_option("excludeSwitches",
["enable-automation", "enable-logging"])
options.add_experimental_option("useAutomationExtension", False)
# CHROME ONLY option to prevent Restore Session popup
options.add_argument("--disable-session-crashed-bubble")
selenium_logger.setLevel(logging_WARNING)
urllib_logger.setLevel(logging_WARNING)
class AnyEc:
"""Use with WebDriverWait to combine expected_conditions
in an OR.
"""
def __init__(self, *args):
self.ecs = args
def __call__(self, driver):
for fn in self.ecs:
try:
if fn(driver):
return True
from splinter import Browser
import datetime
import json
import os
import logging
import time
logging.basicConfig()
logger = logging.getLogger("splinter_tests.__init__")
# Quiet down the Selenium logging
from selenium.webdriver.remote.remote_connection import \
LOGGER as _selenium_logger
_selenium_logger.setLevel(logging.INFO)
SERVER_URL = "http://localhost:5859"
#SERVER_URL = "https://xbeewifi.herokuapp.com"
#SERVER_URL = "https://xbgw-333.herokuapp.com"
def make_url(uri):
return '{}{}'.format(SERVER_URL, uri)
browser = None
driver = "phantomjs"
def _put_local_phantomjs_on_path():
# Use local PhantomJS installation (from node_modules) by putting the path
def run(url):
global wb, ws, driver
# Init start time
start = datetime.datetime.now().replace(microsecond=0)
# Init Log
log_file = open(LOG_FILE, 'w+')
log_file.close()
# Init Driver
LOGGER.setLevel(logging.DEBUG)
driver = init_driver()
if driver is None:
sys.exit()
# Init Excel
wb = openpyxl.Workbook()
ws = wb.active
# TODO: Check if max page exists
# Init Scraping
original_url = url
proceed = True
try:
driver.get(original_url)
title = driver.title
except Exception as e:
log("ERROR Failed openning " + original_url + " - " +
str(e).split(os.linesep, 1)[0])
close_driver()
proceed = False
if proceed is True:
# Get all URLs
ads_urls = []
for page_index in range(1, args.pages + 1):
if driver is None:
driver = init_driver()
if driver is None:
log("WARN Failed to initialize driver, page " +
page_index + " will be skipped")
continue
index = original_url.rfind('/')
url = original_url[:index] + '/page-' + str(
page_index) + original_url[index:]
page_urls = get_page_ads(url, page_index)
if len(page_urls) == 0:
continue
for ad_url in page_urls:
if ad_url not in ads_urls:
ads_urls.append(ad_url)
if (page_index) % 10 == 0 or page_index == args.pages:
close_driver()
# Parse ads
total = len(ads_urls)
log("INFO * Total of " + str(total) + " ads will be scraped")
for index, ad_url in enumerate(ads_urls):
if driver is None:
driver = init_driver()
log("INFO Parsing " + str(index + 1) + "/" + str(total) + ": " +
ad_url)
try:
parse_ad(ad_url)
except selenium.common.exceptions.TimeoutException as e:
log("ERROR Failed to load page " + str(page_index) +
" before time out elapsed, closing driver...")
close_driver()
except Exception as e:
log("ERROR Failed to parse " + ad_url + " - (" +
str(e).split(os.linesep, 1)[0] + ")")
if (index + 1) % 20 == 0 or index + 1 == total:
close_driver()
# Save Results to Excel
try:
if os.path.isfile(RESULTS_FILE):
os.remove(RESULTS_FILE)
wb.save(RESULTS_FILE)
log("INFO Results saved as " + RESULTS_FILE + " in " + DIR_PATH)
except Exception as e:
log("ERROR Failed to save results " + str(e))
# Calculate Duration
end = datetime.datetime.now().replace(microsecond=0)
# Send eMail
if args.mail is True and proceed is True:
if can_send_email():
from_name = "Kijiji Scrapper"
if os.path.isfile(RESULTS_FILE):
message = title + os.linesep
message += original_url + os.linesep
message += "Duration - " + str(end - start) + os.linesep
send_mail(from_name, args.recipients,
"Kijiji Scraping Results", message,
[RESULTS_FILE, LOG_FILE], args.smtp_server,
args.smtp_server_port, args.smtp_server_username,
args.smtp_server_password, True)
else:
message = "Operation Failed." + os.linesep
message += "Logs Attached" + os.linesep
message += "Duration - " + str(end - start)
send_mail(from_name, args.recipients,
"Kijiji Scraping Results - Failed", "Logs attached",
[LOG_FILE], args.smtp_server, args.smtp_server_port,
args.smtp_server_username, args.smtp_server_password,
True)
log("INFO Sent results by mail to: " + ','.join(args.recipients))
else:
log("ERROR Can not send eMail")
log("INFO Script finsihed in " + str(end - start))
# Cleanup
if (args.mail):
if os.path.exists(RESULTS_FILE):
os.remove(RESULTS_FILE)
if os.path.exists(LOG_FILE):
os.remove(LOG_FILE)
print("Bye.")
import logging
import pytest
from selenium.webdriver.remote.remote_connection import LOGGER
LOGGER.setLevel(logging.WARNING)
def test_checkbox_set_to_true(test_page):
test_page.navigate()
test_page.checkbox.set_to(True)
assert test_page.checkbox.checked
def test_checkbox_set_to_false(test_page):
test_page.navigate()
test_page.checkbox.check()
assert test_page.checkbox.checked
test_page.checkbox.set_to(False)
assert test_page.checkbox.checked is False
def test_checkbox_toggle_on(test_page):
test_page.navigate()
test_page.checkbox.toggle()
def download(cfg):
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.remote.remote_connection import LOGGER
LOGGER.setLevel(logging.WARNING)
retCode = False
filename_new = cfg.get('download', 'filename_new')
filename_old = cfg.get('download', 'filename_old')
filename_in = cfg.get('basic', 'filename_in')
# login = cfg.get('download','login' )
# password = cfg.get('download','password' )
url_base = cfg.get('download', 'url_base')
url_file = cfg.get('download', 'url_file')
download_path = os.path.join(os.getcwd(), 'tmp')
if not os.path.exists(download_path):
os.mkdir(download_path)
for fName in os.listdir(download_path):
os.remove(os.path.join(download_path, fName))
dir_befo_download = set(os.listdir(download_path))
if os.path.exists('geckodriver.log'): os.remove('geckodriver.log')
try:
ffprofile = webdriver.FirefoxProfile()
ffprofile.set_preference("browser.download.dir", download_path)
ffprofile.set_preference("browser.download.folderList", 2)
ffprofile.set_preference(
"browser.helperApps.neverAsk.saveToDisk",
",application/octet-stream" + ",application/vnd.ms-excel" +
",application/vnd.msexcel" + ",application/x-excel" +
",application/x-msexcel" + ",application/zip" +
",application/xls" + ",application/vnd.ms-excel" +
",application/vnd.ms-excel.addin.macroenabled.12" +
",application/vnd.ms-excel.sheet.macroenabled.12" +
",application/vnd.ms-excel.template.macroenabled.12" +
",application/vnd.ms-excelsheet.binary.macroenabled.12" +
",application/vnd.ms-fontobject" + ",application/vnd.ms-htmlhelp" +
",application/vnd.ms-ims" + ",application/vnd.ms-lrm" +
",application/vnd.ms-officetheme" +
",application/vnd.ms-pki.seccat" + ",application/vnd.ms-pki.stl" +
",application/vnd.ms-word.document.macroenabled.12" +
",application/vnd.ms-word.template.macroenabed.12" +
",application/vnd.ms-works" + ",application/vnd.ms-wpl" +
",application/vnd.ms-xpsdocument" +
",application/vnd.openofficeorg.extension" +
",application/vnd.openxmformats-officedocument.wordprocessingml.document"
+
",application/vnd.openxmlformats-officedocument.presentationml.presentation"
+
",application/vnd.openxmlformats-officedocument.presentationml.slide"
+
",application/vnd.openxmlformats-officedocument.presentationml.slideshw"
+
",application/vnd.openxmlformats-officedocument.presentationml.template"
+
",application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
+
",application/vnd.openxmlformats-officedocument.spreadsheetml.template"
+
",application/vnd.openxmlformats-officedocument.wordprocessingml.template"
+ ",application/x-ms-application" + ",application/x-ms-wmd" +
",application/x-ms-wmz" + ",application/x-ms-xbap" +
",application/x-msaccess" + ",application/x-msbinder" +
",application/x-mscardfile" + ",application/x-msclip" +
",application/x-msdownload" + ",application/x-msmediaview" +
",application/x-msmetafile" + ",application/x-mspublisher" +
",application/x-msschedule" + ",application/x-msterminal" +
",application/x-mswrite" + ",application/xml" +
",application/xml-dtd" + ",application/xop+xml" +
",application/xslt+xml" + ",application/xspf+xml" +
",application/xv+xml" + ",application/excel")
if os.name == 'posix':
# driver = webdriver.Firefox(ffprofile, executable_path=r'/usr/local/Cellar/geckodriver/0.19.1/bin/geckodriver')
driver = webdriver.Firefox(
ffprofile, executable_path=r'/usr/local/bin/geckodriver')
elif os.name == 'nt':
driver = webdriver.Firefox(ffprofile)
driver.implicitly_wait(10)
driver.set_page_load_timeout(10)
driver.get(url_base)
try:
driver.get(url_file)
except Exception as e:
log.debug('Exception: <' + str(e) + '>')
except Exception as e:
log.debug('Exception: <' + str(e) + '>')
driver.quit()
dir_afte_download = set(os.listdir(download_path))
new_files = list(dir_afte_download.difference(dir_befo_download))
print(new_files)
if len(new_files) == 0:
log.error('Не удалось скачать файл прайса ')
retCode = False
elif len(new_files) > 1:
log.error('Скачалось несколько файлов. Надо разбираться ...')
retCode = False
else:
new_file = new_files[0] # загружен ровно один файл.
new_ext = os.path.splitext(new_file)[-1].lower()
DnewFile = os.path.join(download_path, new_file)
new_file_date = os.path.getmtime(DnewFile)
log.info(
'Скачанный файл ' + new_file + ' имеет дату ' +
time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(new_file_date)))
print(new_ext)
if new_ext in ('.xls', '.xlsx', '.xlsb', '.xlsm', '.csv'):
if os.path.exists(filename_new) and os.path.exists(filename_old):
os.remove(filename_old)
os.rename(filename_new, filename_old)
if os.path.exists(filename_new):
os.rename(filename_new, filename_old)
shutil.copy2(DnewFile, filename_new)
retCode = True
elif new_ext == '.zip':
# ветка устаревшая, не проверялась # Архив. Обработка не завершена
log.debug('Zip-архив. Разархивируем.')
work_dir = os.getcwd()
os.chdir(os.path.join(download_path))
dir_befo_download = set(os.listdir(os.getcwd()))
os.system('unzip -oj ' + new_file)
os.remove(new_file)
dir_afte_download = set(os.listdir(os.getcwd()))
new_files = list(dir_afte_download.difference(dir_befo_download))
os.chdir(work_dir)
if len(new_files) == 1:
new_file = new_files[0] # разархивирован ровно один файл.
new_ext = os.path.splitext(new_file)[-1]
DnewFile = os.path.join(download_path, new_file)
new_file_date = os.path.getmtime(DnewFile)
log.debug('Файл из архива ' + DnewFile + ' имеет дату ' +
time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(new_file_date)))
if os.path.exists(filename_in) and os.path.exists(
'filename_old'):
os.remove('filename_old')
os.rename(filename_in, 'filename_old')
if os.path.exists(filename_in):
os.rename(filename_new, 'filename_old')
shutil.copy2(DnewFile, filename_in)
retCode = True
elif len(new_files) > 1:
log.debug('В архиве не единственный файл. Надо разбираться.')
retCode = False
else:
log.debug(
'Нет новых файлов после разархивации. Загляни в папку юниттеста поставщика.'
)
retCode = False
return retCode
import os, inspect, nose, re, logging
DB_URL_RE = re.compile(r'([^\:]+)\:\/\/([^\:]+)\:([^\@]+)\@([^\/]+)\/([^\s]+)')
environ = {}
if not pylons.test.pylonsapp:
logging.error("Cannot run tests: need to specify config file")
# if this isn't set up right, let it fail in order for nose to display the error
SetupCommand('setup-app').run([pylons.test.pylonsapp.config['__file__']])
try:
from selenium.webdriver.remote.remote_connection import LOGGER
LOGGER.setLevel(logging.INFO)
except ImportError, e:
print "These tests are likely going to fail as Selenium had an import error..."
def CSS(source, selector):
return source.find_elements_by_css_selector(selector)
class SeleniumContextTest(TestCase):
"""
A Selenium test that correctly operates within a QTools context.
Whether or not a subclass of this test runs depends on whether it
is in a directory specified by the 'nose.selenium.test_paths' variable
in the active Pylons configuration.
It will also trigger Firefox as the webdriver to check the front-end
def main(ctx, driver, profile, headless, delay, variance, wait, testrun,
sound):
"""A bot that purchases items online, rendering them bought.
By default, bought uses the parameters specified in config.ini for
configuration but these can also be specified via CLI.
Made with <3 by jsonV
"""
ctx.ensure_object(dict)
ctx.obj["main"] = ctx.params
drive = None
fp = None
no_head = False
eager = False
invoke_newegg = False
wait = 0
logging.basicConfig(
filemode="a",
filename="bought.log",
format="[%(asctime)s] [%(levelname)s] %(message)s",
)
log = logging.getLogger("bought")
log.setLevel(logging.DEBUG)
selenium_logger.setLevel(logging.WARNING)
urllibLogger.setLevel(logging.WARNING)
sh = logging.StreamHandler()
sh.setLevel(logging.DEBUG)
formatter = logging.Formatter("[%(asctime)s] [%(levelname)s] %(message)s",
"%Y-%m-%d %H:%M:%S")
sh.setFormatter(formatter)
log.addHandler(sh)
latest, current = check()
if latest > current:
log.warn(
f"You are running bought v{current}. Please upgrade to v{latest} with: 'pip install --upgrade bought'"
)
# Override defaults if config.ini variable specified
if ctx.config:
driver = ctx.config["Main"]["Driver"].lower()
if ctx.config["Main"]["Headless"].lower() == "true":
no_head = True
if ctx.config["Newegg"]["Enabled"].lower() == "true":
items = ctx.config["Newegg"]["Items"]
invoke_newegg = True
if ctx.config["Main"]["ImplicitWait"]:
wait = int(ctx.config["Main"]["ImplicitWait"])
# Override if CLI argument specified
if driver:
drive = driver
if headless:
no_head = headless
if drive == "firefox":
log.info(f"Using FireFox webdriver.")
if ctx.config:
if os.path.isdir(ctx.config["Main"]["BrowserProfile"]):
log.info(
f'Logging profile: {ctx.config["Main"]["BrowserProfile"]}')
fp = webdriver.FirefoxProfile(
ctx.config["Main"]["BrowserProfile"])
log.info("Finished loading profile.")
else:
log.info("Using default blank profile.")
options = webdriver.FirefoxOptions()
elif drive == "chrome" or drive == "chromium":
log.info(f"Using Chrome webdriver.")
options = webdriver.ChromeOptions()
if ctx.config:
if ctx.config["Main"]["BrowserProfile"]:
log.info(
f'Logging profile: {ctx.config["Main"]["BrowserProfile"]}')
options.add_argument(
f'--user-data-dir={ctx.config["Main"]["BrowserProfile"]}')
else:
log.info("Using default blank profile.")
if no_head:
options.set_headless()
if eager:
options.page_load_strategy = "eager"
if drive == "firefox":
log.info("Starting webdriver.")
driver = webdriver.Firefox(fp, firefox_options=options)
log.info("Successfully created webdriver.")
elif drive == "chrome" or drive == "chromium":
log.info("Starting webdriver.")
driver = webdriver.Chrome(chrome_options=options)
log.info("Successfully created webdriver.")
else:
log.warn("No webdriver specified via config.ini or CLI!")
log.warn(
"Ensure config.ini specifies [Main][Driver] or CLI specifies driver via `bought -d firefox|chrome|chromium!"
)
if wait:
log.info("Setting implicit wait time.")
driver.implicitly_wait(wait)
if drive:
ctx.obj["driver"] = driver
if invoke_newegg:
log.info("Starting Newegg script.")
ctx.obj["config"] = ctx.config
ctx.invoke(newegg, items=items)
from selenium.webdriver.support.ui import WebDriverWait
from anime_downloader.const import get_random_header
from selenium.webdriver.common.by import By
from urllib.parse import urlencode
from urllib.parse import urlsplit
from selenium import webdriver
from bs4 import BeautifulSoup
from logging import exception
from sys import platform
import requests
import os
import logging
import click
import time
import json
serverLogger.setLevel(logging.ERROR)
logger = logging.getLogger(__name__)
def get_data_dir():
'''
Gets the folder directory selescrape will store data,
such as cookies or browser extensions and logs.
'''
APP_NAME = 'anime downloader'
return os.path.join(click.get_app_dir(APP_NAME), 'data')
def open_config():
from anime_downloader.config import Config
return Config
import unittest
import yaml
import time
import os
import roles
from browser import Browser
from ppm_fhir.fhir_services import FHIRServices
import logging
from selenium.webdriver.remote.remote_connection import LOGGER as SeleniumLogger
SeleniumLogger.setLevel(logging.WARNING)
logger = logging.getLogger('ppm-test-suite')
class PPMTestCase(unittest.TestCase):
@classmethod
def setUpClass(cls):
# Set the environment
cls.environment = os.environ.get('PPM_TEST_ENVIRONMENT', 'local')
cls.test_integrations = os.environ.get('PPM_TEST_INTEGRATIONS', False)
# Get the path
path = os.path.dirname(__file__)
# Determine places
with open(os.path.join(path, 'urls.yml'), 'r') as stream:
try:
# import all required modules
from selenium import webdriver
import time, re, hashlib, csv
import pandas as pd
from selenium.webdriver.common.keys import Keys
import usaddress
import logging, datetime, sys
from selenium.webdriver.remote.remote_connection import LOGGER
LOGGER.setLevel(logging.ERROR)
username = '******'
salutation = '' #Mr. Miss
client_name = ''
client_address = ''
billing_card_number = ''
card_cvv_number = ''
billing_address = ''
billing_expiry_date = ''
# log file initialize
logging.basicConfig(
level=logging.DEBUG,
filename='logfile_' + username + '.log', # log to this file
format='%(asctime)s -- %(message)s') # include timestamp
headers = ['Date' , 'Client_Name', 'State', 'County', 'searchterm', 'recordyear', 'last_document_type', \
'last_document_left', 'right_side_name', 'right_side_address', 'right_side_date', 'Doctypes', 'Record_Count', 'last data record found', \
'parsed_rows', 'client_name', 'client_address', 'billing_card_number', 'card_cvv_number', 'billing_address', 'billing_expiry_date']
ignore_keywords = [
'escrow', 'esq', 'inc', 'law', 'service', 'law', 'Timeshare', 'associates',
'recorder', 'recorders', '$', 'iiii', 'service', 'closing', 'title'
def __init__(self, id):
self.endpoint_html = dict()
self.endpoint_soup = dict()
self.id = id
LOGGER.setLevel(logging.WARNING) # remove Selenium logging
import logging
import os
# from pyvirtualdisplay import Display
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.remote.remote_connection import LOGGER
from subprocess import call
import time
import unittest
from webdriver_utils import (
Server,
Client
)
LOGGER.setLevel(logging.WARNING) # silence logging
"""
this setting controls whether the browser will be visible (1) or not (0)
"""
is_visible = 1
# configuration of testing framework
cfg = {
'app': {
'host': '0.0.0.0',
'port': '6543',
'db': 'webdrivertest.db',
'ini': "webdrivertest.ini",
'appSettings': {},
},
def _instantiate_chromedriver(ctx: Context) -> None:
"""Attempt to start the chromedriver, retrying if there are connection errors.
Args:
ctx: The behave context object.
Raises:
:py:class:`.ConnectionResetError`: If starting chromedriver fails too
many times.
"""
# Set the selenium logs to warning only
LOG.setLevel(logging.WARNING)
chrome_options = webdriver.ChromeOptions()
# Prevent images from loading (should decrease load times)
prefs = {"profile.managed_default_content_settings.images": 1}
chrome_options.add_experimental_option("prefs", prefs)
chrome_options.add_argument("--no-sandbox")
# Ignore CORS errors
chrome_options.add_argument("--disable-web-security")
# set logging capability
chrome_options.set_capability("loggingPrefs", {"browser": "ALL"})
# if ctx.zap_proxy_url:
# chrome_options.add_argument(f"--proxy-server={ctx.zap_proxy_url}")
chrome_options.add_argument("--window-size=1920,1080")
# Attempt the connection... Max attempts 3
attempts_remaining = 3
while attempts_remaining > 0:
try:
LOGGER.info("Instantiating chromedriver...")
# use the chromedriver binary loader. this forces the location on path
chromedriver_binary.add_chromedriver_to_path()
ctx.driver = webdriver.Chrome(chrome_options=chrome_options)
LOGGER.debug(
f"Chromedriver running from {chromedriver_binary.chromedriver_filename}"
)
LOGGER.info("Connected to chromedriver successfully!")
break
except (ConnectionResetError, ProtocolError):
# one attempt used...
attempts_remaining -= 1
LOGGER.warning(
"Connection was refused, will try again {} more "
"time{}".format(
attempts_remaining, "" if attempts_remaining == 1 else "s"
)
)
# sleep 3 seconds between attempts
time.sleep(3)
else:
raise ConnectionResetError(
"Failed connecting to chromedriver after exhausting all 3 attempts. Giving up!"
)
# Set the latency for the browser if not defaulted to 0
latency = ctx.latency
if latency != "None":
LOGGER.debug(f"Non default latency was detected as: {latency}")
_set_browser_latency(ctx, latency)
patch_pytest_selenium_screenshot()
def pytest_addoption(parser):
"""Add support for headless option to pytest-selenium."""
parser.addoption('--headless',
action='store',
help='enable headless mode for supported browsers.')
# Disable selenium's remote_connection.py DEBUG statements that pollute
# test stdout.
# They aren't needed as driver logs are included into links of HTML report.
SELENIUM_LOGGER.setLevel(logging.INFO)
@pytest.fixture(autouse=True)
def reset_dict_executed_fixtures():
"""Reset dict_executed_fixtures between tests"""
dynamic_fixtures.dict_executed_fixtures = {}
@pytest.fixture(scope="session")
def session_capabilities(session_capabilities):
"""Log browser (console log) and performance (request / response headers)
data.
They will appear in `Links` section of pytest-html's html report.
"""
session_capabilities["loggingPrefs"] = {
def process_request(self, request, spider):
# Called for each request that goes through the downloader
# middleware.
# Must either:
# - return None: continue processing this request
# - or return a Response object
# - or return a Request object
# - or raise IgnoreRequest: process_exception() methods of
# installed downloader middleware will be called
if (spider.name != 'song_chasing_spidy'
or 'robots.txt' in request.url):
return None
#Init of the selenium webdriver
driver = webdriver.Chrome(
executable_path=ChromeDriverManager().install())
#Selenium logs set to warning
LOGGER.setLevel(logging.WARNING)
#Execute request
driver.get(request.url)
#Wait until links to favorite songs of profile is displayed
link_to_favorites = WebDriverWait(driver, 5).until(
EC.element_to_be_clickable(
(By.XPATH, '//a[contains(@class, "sidebarHeader ")]')))
#Navigate to favorite songs page
link_to_favorites.click()
# Scroll to bottom of the page
# Get scroll height
last_height = driver.execute_script(
"return document.body.scrollHeight")
while True:
# Scroll down to bottom
driver.execute_script(
"window.scrollTo(0, document.body.scrollHeight);")
# Wait to load page
time.sleep(0.5)
# Calculate new scroll height and compare with last scroll height
new_height = driver.execute_script(
"return document.body.scrollHeight")
if new_height == last_height:
break
last_height = new_height
#Set return parameters
body = to_bytes(driver.page_source)
url = driver.current_url
#Close driver
driver.close()
#Return response to spider
return HtmlResponse(url, body=body, encoding='utf-8', request=request)
import os
from logging import WARNING
from splinter.browser import Browser
from selenium.webdriver.remote.remote_connection import LOGGER
LOGGER.setLevel(WARNING)
def before_all(context):
browser = context.config.browser or os.environ.get('browser') or 'firefox'
context.browser = Browser(browser)
def after_all(context):
context.browser.quit()
context.browser = None
import os
import logging
LOG_LEVEL = os.getenv("LOG_LEVEL", 'INFO')
logging.basicConfig(level=LOG_LEVEL)
logging.getLogger().setLevel(LOG_LEVEL)
LOG = logging.getLogger(__name__)
# reduce log levels for selenium. debug is chatty, doesnt default to root level
from selenium.webdriver.remote.remote_connection import LOGGER as serverLogger
serverLogger.setLevel(logging.WARNING)
logging.getLogger("yahoo_oauth").setLevel(level=logging.INFO)
def print_logger_heirarchy(log):
print(
f'{log.name}, effective_level: {log.getEffectiveLevel()}, Type: {type(log)}'
)
if log.parent: print_logger_heirarchy(log.parent)
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
import urlparse
from demerio_utils.log import *
from selenium.webdriver.remote.remote_connection import LOGGER
LOGGER.setLevel(logging.WARNING)
TIME_TO_WAIT = 10
def convert_url_to_query_dict(url):
return dict(urlparse.parse_qsl(urlparse.urlsplit(url).query))
def enter_text_by_id(driver, id, content, press_return=True):
wait = WebDriverWait(driver, TIME_TO_WAIT)
element = wait.until(EC.element_to_be_clickable((By.ID, id)))
element.send_keys(content)
if press_return:
element.send_keys(Keys.RETURN)
def click_button_by_id(driver, id):
wait = WebDriverWait(driver, TIME_TO_WAIT)
btn = wait.until(EC.element_to_be_clickable((By.ID, id)))
btn.click()
def enter_text_by_xpath(driver, xpath, content, press_return=True):
def set_level_logging(level):
urllibLogger.setLevel(level)
LOGGER.setLevel(level)
patch_pytest_selenium_screenshot()
def pytest_addoption(parser):
"""Add support for headless option to pytest-selenium."""
parser.addoption('--headless',
action='store',
help='enable headless mode for supported browsers.')
# Disable selenium's remote_connection.py DEBUG statements that pollute
# test stdout.
# They aren't needed as driver logs are included into links of HTML report.
SELENIUM_LOGGER.setLevel(logging.INFO)
@pytest.fixture(autouse=True)
def reset_dict_executed_fixtures():
"""Reset dict_executed_fixtures between tests"""
dynamic_fixtures.dict_executed_fixtures = {}
@pytest.fixture(scope="session")
def session_capabilities(session_capabilities):
"""Log browser (console log) and performance (request / response headers)
data.
They will appear in `Links` section of pytest-html's html report.
"""
session_capabilities["loggingPrefs"] = {
def setUp(cls, agent=False):
LOGGER.setLevel(logging.WARNING)
cls.driver = cls.create_driver(agent=agent)
from time import sleep
from .utils import (
Utils,
URLS,
TIMEOUTS,
Participant,
tries_n_time_until_true,
names_cleaner,
)
from .exceptions import LoginError
from .interfaces import IModule, IOnAirModule
from selenium.webdriver.remote.remote_connection import LOGGER
import logging
LOGGER.setLevel(logging.WARNING) # reducing selenium verbosity
@retry(stop_max_attempt_number=3)
def _create_hangout_event(browser, name, attendees):
"""
Creates hangout event on google plus. As arguments takes event name and
attendees list, also should be provided with 'browser' object where
visitor is logged in
"""
browser.get(URLS.onair)
browser.by_text('Create a Hangout On Air').click(TIMEOUTS.fast)
# Setting name
browser.xpath(
'//input[@aria-label="Give it a name"]').send_keys(name)
# cleaning 'share' field and send attendees list there
def __init__(self, task_queue, verbose):
'''
Initializes one scraper process
Args:
task_queue: task queue
verbose (bool): print INFO or not
Returns:
Nothing
'''
multiprocessing.Process.__init__(self)
self.task_queue = task_queue
self.verbose = verbose
# init actions
self.web_actions = json.load(open('Configs/actions.json', 'r'))
for i in self.web_actions.keys():
if i[0] is '_':
continue
tmp_file = import_module('Scraper.actions.' + self.web_actions[i])
self.web_actions[i] = getattr(tmp_file, self.web_actions[i])
# inline actions -> flowchart related
self.inline_actions = {
'goto', 'loop', 'auto_get_data'
# other inline actions
}
# init firefox driver
LOGGER.setLevel(logging.WARNING)
self.options = webdriver.FirefoxProfile()
if platform.system() == 'Linux':
# extension
extensions = ['Scraper/driver/ImageBlock.xpi']
self.driver_path = 'Scraper/driver/geckodriver'
# disable Images and Flash
self.options.set_preference('permissions.default.image', 2)
self.options.set_preference(
'dom.ipc.plugins.enabled.libflashplayer.so', 'false')
# add extensions to firefoxx
for ext in extensions:
self.options.add_extension(ext),
self.options.set_preference(
'extensions.ImageBlock.currentVersion', '3.1')
else:
logger(ERR, 'Platform Not Supported')
# run
self.browser = webdriver.Firefox(firefox_profile=self.options,
executable_path=self.driver_path)
self.browser.maximize_window()
try:
# initiate Global event checker
self.gev_checker = event_checker.event_checker(
'Configs/global_events.json', self.browser)
if type(self.gev_checker) is str:
logger(ERR, self.gev_checker)
except:
logger(WARN, 'Cannot Create Event Handler')
self.gev_checker = None
# initiate auto get data
try:
self.auto_getter = auto_get_data('Configs/data_collector.json',
self.browser)
except:
logger(WARN, 'Cannot Create Auto Collector')
self.auto_getter = None
