def run_local(self, browser_version):
"""
Return the local driver
: TODO Add support to other browsers
"""
if command_line.get_browser().lower() == "chrome":
options = chromeOptions()
options.add_argument("disable-infobars")
if command_line.headless() == "y":
options.add_argument("--headless") # For Headless chrome
caps = DesiredCapabilities.CHROME
local_driver = webdriver.Chrome(options=options,
desired_capabilities=caps)
return local_driver
if command_line.get_browser().lower() == "ff" \
or command_line.get_browser().lower() == "firefox":
profile = FirefoxProfile()
options = firefoxOptions()
profile.set_preference("geo.prompt.testing", True)
profile.set_preference("geo.prompt.testing.allow", True)
if command_line.headless() == "y":
options.headless = True # For Headless firefox
local_driver = webdriver.Firefox(
executable_path=GeckoDriverManager().install(),
firefox_profile=profile,
options=options)
return local_driver
def __init__(self,
BROWSER='chrome',
DRIVER='',
HEADLESS=True,
IGNORE_CERT_ERROR=False):
self.driver = None
self.id = None
self.password = None
self.loginflag = False
fp = None
options = None
if BROWSER == 'firefox':
options = firefoxOptions()
fp = self.get_firefox_profile()
elif BROWSER == 'chrome':
options = chromeOptions()
if HEADLESS == True:
options.set_headless(headless=True)
if IGNORE_CERT_ERROR == True:
options.add_argument('--ignore-certificate-errors')
if BROWSER == 'firefox':
self.driver = webdriver.Firefox(firefox_binary=FIREFOX_BINARY,
firefox_options=options,
log_path='/dev/null',
firefox_profile=fp)
elif BROWSER == 'chrome':
if len(DRIVER):
self.driver = webdriver.Chrome(DRIVER, chrome_options=options)
else:
self.driver = webdriver.Chrome(chrome_options=options)
def prepare_webdriver():
'''Returns a WebDriver for the script execution'''
options = firefoxOptions()
options.add_argument('-headless')
options.binary_location='/usr/bin/firefox'
driver = Firefox(executable_path='geckodriver', options=options)
# options = chromeOptions()
# options.add_argument('--headless')
# options.binary_location = '/usr/bin/chromium'
# driver = Chrome(executable_path='chromedriver', options=options)
print('[!] Initializing WebDriver...')
wait = WebDriverWait(driver, timeout=4)
print('[!] WebDriver ready!')
return driver
def __init__(self, driver=None):
'''
Instantiate NoVNCbot for interacting with a VNC Console
@param driver: Optional override that accepts the following values
String: firefox, ff -- Creates Firefox browser in headless mode
String: chrome -- Creates chrome browser in headless mode
WebDriver: -- Uses webdriver created by caller
None: Creates Firefox browser in headless mode
'''
if isinstance(driver, WebDriver):
self.driver = driver
elif isinstance(driver, str) and (driver.lower() == 'firefox'
or driver.lower() == 'ff'):
firefox_options = firefoxOptions()
firefox_options.headless = True
self.driver = webdriver.Firefox(options=firefox_options)
elif isinstance(driver, str) and (driver.lower() == 'chrome'):
chrome_options = chromeOptions()
chrome_options.add_argument("--headless")
self.driver = webdriver.Chrome(options=chrome_options)
else:
default_options = firefoxOptions()
default_options.headless = True
self.driver = webdriver.Firefox(options=default_options)
def setup_driver_options(self, user_agent):
if self.driver_type == 'firefox':
firefox_options = firefoxOptions()
firefox_options.add_argument("--window-size=1420,1080")
# firefox_options.add_argument("--headless")
firefox_options.add_argument("--disable-gpu")
firefox_options.add_argument(f'user-agent={user_agent}')
# capabilities = self.setup_proxy(driver_type.lower())
return firefox_options
elif self.driver_type == 'chrome': #add more driver types (chrome, etc if necessary)
chrome_options = chromeOptions()
# chrome_options.add_argument("--headless")
# chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument("--window-size=1420,1080")
chrome_options.add_argument("--disable-gpu")
chrome_options.add_argument(f'user-agent={user_agent}')
return chrome_options
else:
print("not supported driver type")
def python_selenium_test(browser_var='chrome'):
print('Hello, world!')
if ('headless' in browser_var.lower()):
if ('chrome' in browser_var.lower()):
chrome_options = chromeOptions()
chrome_options.add_argument("--headless")
browser = webdriver.Chrome(chrome_options=chrome_options)
elif ('firefox' in browser_var.lower()):
firefox_options = firefoxOptions()
firefox_options.add_argument("--headless")
browser = webdriver.Firefox(options=firefox_options)
else:
if ('chrome' in browser_var.lower()):
browser = webdriver.Chrome()
elif ('firefox' in browser_var.lower()):
browser = webdriver.Firefox()
wait = ui.WebDriverWait(browser, 5)
browser.get('https://the-internet.herokuapp.com/login')
wait.until(
expected_conditions.visibility_of_element_located((By.ID, 'username')))
browser.find_element_by_id('username').click()
browser.find_element_by_id('username').send_keys('tomsmith')
wait.until(
expected_conditions.visibility_of_element_located((By.ID, 'password')))
browser.find_element_by_id('password').click()
browser.find_element_by_id('password').send_keys('SuperSecretPassword!')
browser.find_element_by_css_selector('button[type=submit]').click()
wait.until(lambda browser: browser.find_element_by_css_selector(
'.button > i.icon-signout'))
time.sleep(5)
browser.quit()
def create_driver(self):
cur_config = self.options._config
# get executable driver absolute path
folder_path = ""
if cur_config['folder_path'] != "":
folder_path = cur_config['folder_path']
else:
folder_path = path.join(path.dirname(path.realpath(__file__)),
cur_config['browser_name'])
file_lists = listdir(folder_path)
self.driver_path = ""
for file_item in file_lists:
if file_item[-3:] == 'exe':
self.driver_path = path.join(folder_path, file_item)
if self.driver_path == "":
self.print_log("driver name not found")
return None
else:
self.print_log("your driver exe path is {0}".format(
self.driver_path))
if cur_config['browser_name'] == "chrome":
# settings for chrome
chrome_options = Options()
# Private browsing
chrome_options.add_argument("--incognito")
if cur_config['is_maximized']:
chrome_options.add_argument("start-maximized")
# for automation
chrome_options.add_experimental_option("excludeSwitches",
["enable-automation"])
chrome_options.add_experimental_option('useAutomationExtension',
False)
# javascript or image loaded option
if not cur_config['js_loaded']:
chrome_options.add_experimental_option(
"prefs",
{'profile.managed_default_content_settings.javascript': 2})
if not cur_config['img_loaded']:
chrome_options.add_experimental_option(
"prefs",
{"profile.managed_default_content_settings.images": 2})
# To use my default extension
# user_data_path = "C:\\Users\\" + getlogin() + "\\AppData\\Local\\Google\\Chrome\\User Data"
# user_data_setting = "user-data-dir=" + user_data_path
# chrome_options.add_argument(user_data_setting)
if cur_config['used_proxy']:
self.print_log("current proxy ip is : " +
cur_config['cur_proxy'])
chrome_options.add_argument('--proxy-server=' +
cur_config['cur_proxy'])
else:
self.print_log("Now opening Google Chrome with no proxy...")
driver = wd.Chrome(executable_path=self.driver_path,
options=chrome_options)
elif cur_config['browser_name'] == "firefox":
# create firefox options
options = firefoxOptions()
firefox_profile = wd.FirefoxProfile()
if not cur_config['js_loaded']:
options.preferences.update({'javascript.enabled': False})
if not cur_config['img_loaded']:
firefox_profile.set_preference('permissions.default.image', 2)
firefox_profile.set_preference(
'dom.ipc.plugins.enabled.libflashplayer.so', 'false')
if cur_config['used_proxy']:
self.print_log("current proxy ip is : " +
cur_config['cur_proxy'])
# set proxy
firefox_capabilities = wd.DesiredCapabilities.FIREFOX
firefox_capabilities['marionette'] = True
firefox_capabilities['proxy'] = {
'proxyType': "MANUAL",
'httpProxy': cur_config['cur_proxy'],
'ftpProxy': cur_config['cur_proxy'],
'sslProxy': cur_config['cur_proxy']
}
self.print_log("Now opening Firefox browser...")
driver = wd.Firefox(executable_path=self.driver_path,
capabilities=firefox_capabilities,
options=options,
firefox_profile=firefox_profile)
else:
self.print_log("Now opening Firefox browser with no proxy...")
driver = wd.Firefox(executable_path=self.driver_path,
options=options,
firefox_profile=firefox_profile)
if cur_config['is_maximized']:
driver.maximize_window()
else:
self.print_log("Unrecognized Browser")
return None
if cur_config['window_pos'] != [0, 0]:
driver.set_window_position(cur_config['window_pos'][0],
cur_config['window_pos'][1])
return driver
def get_initinfo(url, proxy, sleep_time):
def isElementPresent(driver, value):
try:
element = driver.find_element_by_class_name(value)
# 原文是except NoSuchElementException, e:
except Exception as e:
# 打印异常信息
print(e)
# 发生了NoSuchElementException异常，说明页面中未找到该元素，返回False
return False
# 没有发生异常，表示在页面中找到了该元素，返回True
return True
"""
本函数用于获取签名sign信息,transactionID和后续请求data.
其中使用了selenium和browsermob-proxy.
参数：
url: 携程搜索国际航班的url
返回值：
headers：后续请求头信息
postdata: 后续持续获取航班信息请求头中的提交json信息
"""
# chrome测试配置
# options = Options()
# options.add_argument('--ignore-certificate-errors')
# options.add_argument('--proxy-server={0}'.format(proxy.proxy))
# options.add_argument('--disable-gpu')
# options.add_argument('window-size=1200x600')
# # options.add_argument('--headless')
# driver = webdriver.Chrome(options=options) # 使用selenium创建浏览器窗口
# firefox测试配置
options_ = firefoxOptions()
options_.add_argument('--ignore-certificate-errors')
options_.add_argument('--proxy-server={0}'.format(proxy.proxy))
options_.add_argument('--disable-gpu')
options_.add_argument('--headless')
profile = webdriver.FirefoxProfile()
profile.set_proxy(proxy.selenium_proxy())
# profile.set_preference('permissions.default.image',2)
driver = webdriver.Firefox(options=options_, firefox_profile=profile)
proxy.new_har(url,
options={
'captureContent': True,
'captureHeaders': True
}) # 代理服务器开始监测，捕捉文本和请求头信息
# 显示等待5秒，因为网页会持续加载，需要等待一段时间，直到航空公司内容出现，说明加载成功
driver.set_page_load_timeout(10)
try:
driver.get(url)
except Exception as e:
print(e)
driver.quit()
return "timeout", ''
time.sleep(5)
try:
WebDriverWait(driver, sleep_time, 0.5).until(
EC.presence_of_element_located(
(By.CLASS_NAME, 'airline-name'))) #airline-name
except Exception as e:
if isElementPresent(driver, 'no-result'):
return "empty", ''
else:
print(e)
return "error_timeout", ''
finally:
driver.quit()
result = proxy.har
# 获取https://flights.ctrip.com/international/search/api/search/batchSearch这个访问过程中的重要信息
headers = {}
for entry in result['log']['entries']:
if 'batchSearch' in entry['request']['url']:
postdata = entry['request']['postData']['text']
header = entry['request']['headers']
# response = entry['response']['content']['text']
# response_ = json.loads(response)['data']
# if 'flightItineraryList' in response_:
# result = response_['flightItineraryList']
# else:
# result = "empty"
for x in header:
if x['name'] == 'Connection':
headers[x['name']] = 'close'
else:
headers[x['name']] = x['value']
# return result
return headers, postdata
return "error", ''