def __init__(self, email, username, password, proxy=None):
options = webdriver.ChromeOptions()
options.add_experimental_option("excludeSwitches", ["enable-logging"])
self.seleniumwire_options = {'proxy': {
'http': proxy,
'https': proxy
}
}
"""if proxy:
options.add_argument('--proxy-server=%s' % proxy)"""
self.platform = platform.system() #Support for linux system
"""if self.platform == 'Windows':
self.driver = webdriver.Chrome(options=options, executable_path=r"chromedriver.exe")
else:
self.driver = webdriver.Chrome(options=options, executable_path=r"chromedriver")"""
#Webdrivermanager install chromedriver binary as per system's platform
if proxy:
#check if proxy is working and also update timezone as per proxy to avoid detection
try:
self.timezone = requests.get('http://ip-api.com/json',proxies = self.seleniumwire_options['proxy'],timeout = 10).json()['timezone']
self.tz_params = {'timezoneId': self.timezone}
self.driver = webdriver.Chrome(options = options,executable_path = ChromeDriverManager().install(),seleniumwire_options=self.seleniumwire_options)
except:
self.timezone = None
free_print(f"{Fore.LIGHTMAGENTA_EX}[!]{Style.RESET_ALL} " + 'Skipping Proxy as Connection Issue Have you entered it in the correct format?')
self.driver = webdriver.Chrome(options = options,executable_path = ChromeDriverManager().install())
else:
self.timezone = None
self.driver = webdriver.Chrome(options = options,executable_path = ChromeDriverManager().install())
self.email= email
self.username = username
self.password = password
def create_chromedriver(self):
"""
Since we have many functions we can perform we consolidate
chromedriver code here.
"""
# set up options object
chrome_options = Options()
# if we have chrome binary set it up
if self.chrome_binary_path:
chrome_options.binary_location = self.chrome_binary_path
# live dangerously
if self.allow_insecure:
chrome_options.add_argument('--allow-running-insecure-content')
# thank god for this option
chrome_options.add_argument('--mute-audio')
# if we are headless we also mix up window size a bit
if self.headless:
chrome_options.add_argument('headless')
chrome_options.add_argument('disable-gpu')
window_x = random.randrange(1050, 1920)
window_y = random.randrange(900, 1080)
chrome_options.add_argument('window-size=%sx%s' %
(window_x, window_y))
# if we have a ua set it here
if self.ua:
chrome_options.add_argument('user-agent=' + self.ua)
# attempt to start driver, fail gracefull otherwise
try:
# if we have chromedriver path set it up
if self.chromedriver_path:
driver = webdriver.Chrome(self.chromedriver_path,
desired_capabilities={
'loggingPrefs': {
'performance': 'ALL'
}
},
chrome_options=chrome_options)
else:
driver = webdriver.Chrome(
desired_capabilities={
'loggingPrefs': {
'performance': 'ALL'
}
},
chrome_options=chrome_options,
)
except:
return None
# allow one minute before we kill it, seperate from browser_wait
driver.set_page_load_timeout(self.page_timeout_seconds)
return driver
def __buildDriver__(self, driver_options):
# Ubuntu
if (os.name == 'posix'):
chromedriver = 'chromedriver'
# Windows
if (os.name == 'nt'):
chromedriver = 'chromedriver.exe'
if chromedriver:
if config.DRIVER_NAME == 'Chrome':
logger.info("Using Chrome Driver ...")
options = webdriver.ChromeOptions()
options.add_experimental_option("excludeSwitches",
["enable-automation"])
options.add_experimental_option('useAutomationExtension',
False)
options.add_experimental_option('w3c', False)
for driver_option in driver_options:
# if "--proxy-server" in driver_option:
# print(f"\nADDING PROXY: [{driver_option}]\n")
options.add_argument(driver_option)
capabilities = None
if config.PROXY:
# print(f"\nADDING PROXY: [{config.PROXY}]\n")
# prox = Proxy()
# prox.proxy_type = ProxyType.MANUAL
# prox.http_proxy = config.PROXY
# # prox.socks_proxy = config.PROXY
# prox.ssl_proxy = config.PROXY
capabilities = webdriver.DesiredCapabilities.CHROME
capabilities['loggingPrefs'] = {'performance': 'ALL'}
# prox.add_to_capabilities(capabilities)
if capabilities:
self.driver = webdriver.Chrome(
desired_capabilities=capabilities, options=options)
# self.driver = config.DRIVER_NAME(desired_capabilities=capabilities, options=options)
else:
self.driver = webdriver.Chrome(chromedriver,
options=options)
# self.driver = config.DRIVER_NAME(chromedriver, options=options)
else:
logger.info("Using Firefox Driver ...")
self.driver = webdriver.Firefox()
self.min_wait = WebDriverWait(self.driver, 5)
self.max_wait = WebDriverWait(self.driver, 20)
self.els_css = self.driver.find_elements_by_css_selector
self.el_css = self.driver.find_element_by_css_selector
self.els_xpath = self.driver.find_elements_by_xpath
self.el_xpath = self.driver.find_element_by_xpath
self.driver.set_script_timeout(30)
return self.driver
def set_driver_for_browser(self, browser_name):
"""expects browser name and returns a driver instance"""
# if browser is suppose to be chrome
if browser_name.lower() == "chrome":
browser_option = ChromeOptions()
# automatically installs chromedriver and initialize it and returns the instance
if self.proxy is not None:
options = {
'https': 'https://{}'.format(self.proxy.replace(" ", "")),
'http': 'http://{}'.format(self.proxy.replace(" ", "")),
'no_proxy': 'localhost, 127.0.0.1'
}
print("Using: {}".format(self.proxy))
return webdriver.Chrome(executable_path=ChromeDriverManager().install(),
options=self.set_properties(browser_option), seleniumwire_options=options)
return webdriver.Chrome(executable_path=ChromeDriverManager().install(), options=self.set_properties(browser_option))
elif browser_name.lower() == "firefox":
browser_option = FirefoxOptions()
if self.proxy is not None:
options = {
'https': 'https://{}'.format(self.proxy.replace(" ", "")),
'http': 'http://{}'.format(self.proxy.replace(" ", "")),
'no_proxy': 'localhost, 127.0.0.1'
}
print("Using: {}".format(self.proxy))
return webdriver.Firefox(executable_path=GeckoDriverManager().install(),
options=self.set_properties(browser_option), seleniumwire_options=options)
# automatically installs geckodriver and initialize it and returns the instance
return webdriver.Firefox(executable_path=GeckoDriverManager().install(), options=self.set_properties(browser_option))
else:
# if browser_name is not chrome neither firefox than raise an exception
raise Exception("Browser not supported!")
def __init__(self, proxy_class, price_class):
"""Создаем экземпляр бота"""
Thread.__init__(self)
if settings.DEBUG:
print('CREATE BOT')
self.proxies = proxy_class
self.price = price_class
self.no_proxy = False
self.use_proxy = 1
if self.use_proxy == 2:
self.proxy = ''
else:
self.proxy, clean_proxy = self.proxies.get_new_proxy()
"""Подключаем настройки прокси к экземпляру бота"""
options = {'proxy': self.proxy}
wind_width = random.randint(1000, 1920)
wind_height = random.randint(750, 1080)
with open(
f'{settings.MEDIA_DIR}/user-agents/user_agents_for_chrome_pk.txt',
'r',
encoding='utf-8') as f:
user_agent = random.choice([line.strip() for line in f])
self.options = webdriver.ChromeOptions()
self.options.add_argument("ignore-certificate-errors")
self.options.add_argument(f'window-size={wind_width},{wind_height}')
self.options.add_argument("--disable-blink-features")
self.options.add_argument(
"--disable-blink-features=AutomationControlled")
self.options.add_argument("--headless")
self.options.add_argument('--disable-extensions')
self.options.add_argument('--profile-directory=Default')
"""Добавялем юзер агент из списка"""
self.options.add_argument(f"user-agent={user_agent}")
"""Создаем экземпляр браузера"""
if self.use_proxy == 1:
self.driver = webdriver.Chrome(
options=self.options,
executable_path=f'{settings.MEDIA_DIR}/drivers/chromedriver',
seleniumwire_options=options,
)
else:
self.driver = webdriver.Chrome(
options=self.options,
executable_path=f'{settings.MEDIA_DIR}/drivers/chromedriver',
)
"""Создаем необходимые переменные для работы"""
self.reboot = False
self.search_status = False
self.step = 0
self.target = []
def get_webdriver(headless=False, executable_path=r'./chromedriver.exe'):
if headless:
options = Options()
options.add_argument("--headless")
driver = wire_webdriver.Chrome(options=options)
else:
#driver = webdriver.Chrome('C:\\Users\\caioc\\Desktop\\scripts\\chromedriver')
driver = wire_webdriver.Chrome(executable_path=executable_path)
return driver
def select_driver(browser):
if browser == "Chrome":
return webdriver.Chrome(executable_path='/usr/local/bin/chromedriver',
seleniumwire_options={'verify_ssl': False})
# return webdriver.Chrome(executable_path='/usr/local/bin/chromedriver')
elif browser == "Firefox":
return webdriver.Firefox()
else:
return webdriver.Chrome(executable_path='/usr/local/bin/chromedriver')
def getFederatedRequestParams(username, password, samlUrl):
import selenium
from seleniumwire import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.options import Options
import time
from urllib.parse import unquote
from urllib.parse import quote
# add some headless options
options = Options()
options.headless = True
options.add_experimental_option("excludeSwitches",["ignore-certificate-errors"])
# using chrome to access web pages
driver = webdriver.Chrome(chrome_options=options)
# Open federated authentication webpage that returns the needed saml response
driver.get(samlUrl)
# select the username box when it to loads
wait = WebDriverWait(driver, 10)
id_box = wait.until(EC.element_to_be_clickable((By.ID, 'userNameInput')))
# enter username
id_box.send_keys(username)
# select password box
pass_box = driver.find_element_by_id('passwordInput')
# enter password
pass_box.send_keys(password)
# select Sign in box
signIn_button = driver.find_element_by_id('submitButton')
# click Sign in button
signIn_button.click()
# find the proper response header and use it to get the bearer token
for request in driver.requests:
if request.response and request.path == "https://anypoint.mulesoft.com/accounts/login/receive-id":
# convert from binary to string and strip leading text
samlResponse = unquote(str(request.body,'utf-8'))[13:]
# get the bearer token
url = 'https://anypoint.mulesoft.com/accounts/login/receive-id'
body = '{"SAMLResponse": "%s"}' % samlResponse
headers = {
'Content-Type': 'application/json',
'X-Requested-With': 'XMLHttpRequest'
}
# close the chrome browser instance
driver.quit()
return url, headers, body
def __init__(self, url):
self.url = url
#self.options = webdriver.ChromeOptions()
#self.options.add_argument('headless') # для открытия headless-браузера
#self.driver= webdriver.Chrome(options=self.options)
self.driver = webdriver.Chrome()
self.navigate()
def get_chrome_driver(driver_path):
"""
Instantiates a Chrome webdriver object
Args:
driver_path: path to the Chrome driver
Returns:
webdriver: returns a webdriver object
"""
options = webdriver.ChromeOptions()
options.page_load_strategy = 'normal'
options.add_argument("--disable-logging")
options.add_argument("--disable-login-animations")
options.add_argument("--disable-notifications")
options.add_argument("--disable-default-apps")
options.add_argument("--disable-extensions")
options.add_experimental_option("excludeSwitches",
["ignore-certificate-errors"])
options.add_argument("--disable-dev-shm-usage")
options.add_argument("--disable-browser-side-navigation")
options.add_argument("--headless")
options.add_argument("--hide-scrollbars")
options.add_argument('--log-level 3')
options.add_argument("--incognito")
options.add_argument("--no-zygote")
options.add_argument('--disable-gpu')
options.add_argument('--no-sandbox')
options.add_argument('--disable-infobars')
driver = webdriver.Chrome(driver_path, options=options)
driver.set_page_load_timeout(PAGE_LOAD_TIMEOUT_SEC)
return driver
def post(self, request):
try:
if request.is_ajax():
command = request.POST.get('command')
if command == 'start':
url = request.POST.get('url')
scopes = request.POST.getlist('scopes[]')
options = {
'cleanup_onexit': False,
'connection_timeout': None # Never timeout
}
# Download and install Chrome web driver
global driver
driver = webdriver.Chrome(
ChromeDriverManager().install(), seleniumwire_options=options)
# Specifying driver scopes
driver.scopes = scopes
# Opening up the URL address
driver.get(url)
#p1 = multiprocessing.Process(target=mytest)
# p1.start()
return JsonResponse({'status': 'success'}, status=200)
else:
return JsonResponse({'status': 'success'}, status=200)
except:
return JsonResponse({'response': 'error'}, status=200)
def get_hangouts_tokens(cookies, driverpath):
''' gets auth and hangout token '''
tmprinter = TMPrinter()
chrome_options = get_chrome_options_args(config.headless)
options = {
'connection_timeout': None # Never timeout, otherwise it floods errors
}
tmprinter.out("Starting browser...")
driver = webdriver.Chrome(executable_path=driverpath,
seleniumwire_options=options,
options=chrome_options)
driver.header_overrides = config.headers
tmprinter.out("Setting cookies...")
driver.get("https://hangouts.google.com/robots.txt")
for k, v in cookies.items():
driver.add_cookie({'name': k, 'value': v})
tmprinter.out("Fetching Hangouts homepage...")
driver.get("https://hangouts.google.com")
tmprinter.out("Waiting for the /v2/people/me/blockedPeople request, it "
"can takes a few minutes...")
req = driver.wait_for_request('/v2/people/me/blockedPeople', timeout=120)
tmprinter.out("Request found !")
driver.close()
tmprinter.out("")
auth_token = req.headers["Authorization"]
hangouts_token = req.url.split("key=")[1]
return (auth_token, hangouts_token)
def load_browser():
global driver
global ublock_guid
chrome_options = webdriver.ChromeOptions()
# chrome_options.binary_location = "/applications/developer/google\ chrome.app/Contents/MacOS/Google\ Chrome"
# chrome_options.add_argument("--disable-web-security") # messes up ublock
# chrome_options.add_argument('--headless')
chrome_options.add_extension('ubo_1_30_4_0.crx')
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--disable-gpu')
chrome_options.add_argument("--ignore-certificate-errors")
chrome_options.add_argument("--enable-javascript")
chrome_options.add_argument("--disable-chrome-google-url-tracking-client")
chrome_options.add_argument("--safebrowsing-disable-download-protection")
chrome_options.add_argument("--disable-domain-reliability")
chrome_options.add_argument("--allow-running-insecure-content")
chrome_options.add_argument(
"--unsafely-treat-insecure-origin-as-secure=http://host:port")
chrome_options.add_argument(
'--user-agent=Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Safari/537.36'
)
driver = webdriver.Chrome(executable_path="./chromedriver",
options=chrome_options)
driver.header_overrides = {'Referer': 'com.google.android.gm'}
# display = Display(visible=0, backend="xephyr", size=(800, 600))
# display.start()
extension_uri = "chrome-extension://cjpalhdlnbpafiamejdnhcphjbkeiagm/logger-ui.html?popup=1#_"
driver.implicitly_wait(page_delay)
driver.get(extension_uri)
ublock_guid = driver.current_window_handle
def test_proxy(proxy,url,page_title,use_request=False):
host = proxy[0]
port = proxy[1]
username = proxy[2]
password = proxy[3]
if use_request == True:
proxy_argument = {'https': f'https://{username}:{password}@{host}:{port}'}
try:
print("Testing proxy")
r = requests.get(f'https://{url}', proxies=proxy_argument, timeout=3.5)
print('Found working proxy')
return proxy
except requests.exceptions.ProxyError:
print(f'{proxy} Banned on {url}')
else:
options = {
'proxy': {
'https': f'https://{username}:{password}@{host}:{port}',
}
}
browser = webdriver.Chrome("/Users/CreeperFire/Desktop/PythonT/chromedriver", seleniumwire_options=options)
try:
browser.get(url)
browser.implicitly_wait(5)
if page_title in browser.title == True:
return proxy
except OSError or selenium.common.exceptions.TimeoutException:
print(proxy,"banned")
browser.close()
def __init__(self):
self.browser = webdriver.Chrome(chromedriver,
options=chrome_options,
seleniumwire_options=proxy_options)
self.telegram = TelegramBot()
self.max_likes = 20
self.likes_count = 0
def get_driver(*args, **kwargs):
options = Options()
options.headless = False
options.add_argument("--window-size=1920,1080")
options.add_argument('--no-sandbox')
options.add_argument('--disable-dev-shm-usage')
options.add_argument('--disable-crash-reporter')
options.add_argument('--disable-logging')
options.add_argument('--log-level=3')
options.add_argument('--ignore-ssl-errors=yes')
options.add_argument('--ignore-certificate-errors')
options.add_argument(
'--user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.55 Safari/537.36'
)
if platform.system() == 'Linux':
DRIVER_PATH = 'chromedriver'
elif platform.system() == "Darwin":
DRIVER_PATH = (pathlib.Path(__file__).parent.parent /
'chromedriver').resolve()
else:
log.error('Unsupported OS')
exit(0)
# TODO find a good proxy to use so that this works from Azure
proxy_options = {'auto_config': False, 'disable_capture': True}
driver = webdriver.Chrome(options=options,
executable_path=DRIVER_PATH,
seleniumwire_options=proxy_options)
yield driver
driver.close()
driver.quit()
def init_chrome_driver(driver_path=None) -> WebDriver:
"""
Initialize a Chrome webdriver
with custom headers support
Assumes Chrome Driver path executable is set by
env :: CHROME_DRIVER_PATH, defaults to /usr/local/bin
"""
if driver_path is None:
driver_path = os.environ.get('CHROME_DRIVER_PATH',
'/usr/local/bin/chromedriver')
logger.info(f"Chrome Driver path at '{driver_path}'")
options = Options()
options.add_argument("--headless")
options.add_argument("--window-size=1920x1080")
options.add_argument('--log-level 3')
logger.debug(f"Chrome options: '{options}'")
logger.debug(json.dumps(options.to_capabilities(), indent=4))
driver = webdriver.Chrome(chrome_options=options,
executable_path=driver_path)
return driver
def __init__(
self,
options=None,
user_agent=None,
wait_time=2,
fullscreen=False,
incognito=False,
headless=False,
**kwargs
):
options = Options() if options is None else options
if user_agent is not None:
options.add_argument("--user-agent=" + user_agent)
if incognito:
options.add_argument("--incognito")
if fullscreen:
options.add_argument("--kiosk")
if headless:
options.add_argument("--headless")
options.add_argument("--disable-gpu")
self.driver = webdriver.Chrome(options=options, **kwargs)
self.element_class = WebDriverElement
self._cookie_manager = CookieManager(self.driver)
super().__init__(wait_time)
def crawlWebsite():
print('[ INFO ] ' + 'Crawling from ' + base_url)
driver = webdriver.Chrome('/usr/local/bin/chromedriver',
seleniumwire_options=options,
options=chrome_options)
driver.implicitly_wait(5)
driver.get(base_url)
more_button = driver.find_elements_by_xpath(MUAT_LAINNYA_XPATH)
btn_counter = 0
while (len(more_button) > 0) and btn_counter < 100:
more_button[0].click()
btn_counter += 1
print('[ INFO ] Clicked MUAT LAINNYA! Number of clicks: ' +
str(btn_counter))
time.sleep(2)
more_button = driver.find_elements_by_xpath(MUAT_LAINNYA_XPATH)
print('[ INFO ] No more MUAT LAINNYA button. Oh boy time to parse!')
page = soup(driver.page_source, 'html.parser')
car_tags = page.find_all(class_='EIR5N')
car_data = []
print('[ INFO ] Total link: ' + str(len(car_tags)))
for cars in car_tags:
car_link = cars.find('a', href=True)
car_url = car_url_prefix + car_link['href']
value_dict = crawlCarData(car_url)
if (bool(value_dict)):
car_data.append(value_dict)
percentage = str(int(
(len(car_data) / len(car_tags)) * 100)).rjust(3)
print('[ ' + percentage + '% ] ' + 'Crawled from ' + car_url)
time.sleep(1)
driver.close()
print('[ INFO ] Crawling finished. Total data crawled: ' +
str(len(car_data)))
return car_data
def initiate_driver(url):
global driver
#options = webdriver.ChromeOptions()
#options.add_experimental_option('debuggerAddress', 'localhost:9014')
driver = webdriver.Chrome(executable_path="chromedriver.exe",
seleniumwire_options={'port': 9014})
driver.get(url)
def driver(self):
if self._driver is None:
options = self.options()
scopes = self.scopes()
self._driver = webdriver.Chrome(options=options)
self._driver.scopes = scopes
return self._driver
def initialize_web_driver(self):
# Add additional Options to the webdriver
chrome_options = ChromeOptions()
# add the argument and make the browser Headless.
if self.headless_browser:
chrome_options.add_argument("--headless")
"""
chrome_options.add_argument("--ignore-certificate-errors")
chrome_options.add_argument("--incognito")
chrome_options.add_argument("--kiosk")
"""
# set options
"""
chrome_options.add_argument("--window-size=1920,1080")
chrome_options.add_argument("--disable-gpu")
chrome_options.add_argument("--disable-extensions")
chrome_options.add_argument("--proxy-server='direct://'")
chrome_options.add_argument("--proxy-bypass-list=*")
chrome_options.add_argument("--start-maximized")
"""
driver = webdriver.Chrome(
executable_path='E:/chromedriver_win32/chromedriver.exe',
options=chrome_options)
print('got driver')
return driver
def __init__(self, frontier_lock, access_time_lock):
self.USER_AGENT = "fri-wier-agmsak"
self.WEB_DRIVER_LOCATION = "chromedriver"
self.wait_for = 5
self.delay = 5
self.frontier_lock = frontier_lock
self.access_time_lock = access_time_lock
self.DEBUG = False
self.site_table = tables.SiteTable()
self.page_table = tables.PageTable()
self.image_table = tables.ImageTable()
self.ip_table = tables.SiteIPAddrTable()
self.link_table = tables.LinkTable()
self.pagedata_table = tables.PageDataTable()
self.current_page_id = None
self.current_url = None
self.processing_page = None
options = Options()
options.add_argument("--headless")
options.add_argument("user-agent=" + self.USER_AGENT)
self.driver = webdriver.Chrome(self.WEB_DRIVER_LOCATION,
options=options)
def init_driver(self):
# settings selenium
driver_options = webdriver.ChromeOptions()
if self.browser_executable_path:
driver_options.binary_location = self.browser_executable_path
# # switch user_agent, proxy
# if hasattr(self, 'agent'):
# driver_options.add_argument(f"--user-agent='{next(self.agent)}'")
# if hasattr(self, 'proxy'):
# driver_options.add_argument(f"--proxy-server='{next(self.proxy)}'")
# add SELENIUM_DRIVER_ARGUMENTS options
for argument in self.driver_arguments:
driver_options.add_argument(argument)
# add experimental option: on/off image and notify
chrome_prefs = {"disk-cache-size": 4096}
if not self.driver_load_img:
chrome_prefs.update(
{
"profile.default_content_settings": {"images": 2},
"profile.managed_default_content_settings": {"images": 2}
}
)
if self.driver_disable_notify:
chrome_prefs.update(
{"profile.default_content_setting_values.notifications": 2}
)
driver_options.add_experimental_option("prefs", chrome_prefs)
self.driver = webdriver.Chrome(
executable_path=self.driver_executable_path,
options=driver_options
)
def get_info(link):
reviews = []
options = 1
driver = 1
try:
options = webdriver.ChromeOptions()
options.add_argument('--ignore-certificate-errors')
options.add_argument('--incognito')
options.add_argument('--headless')
driver = webdriver.Chrome(executable_path=binary_path, chrome_options=options)
driver.get(link)
wait = WebDriverWait(driver,120)
wait.until(wait_requests)
except TimeoutException:
print("Failed to load search bar at www.google.com")
except Exception as e:
print(e)
finally:
for request in driver.requests:
if request.response and request.response.headers['Content-Type'] == 'application/json' and request.path == 'https://www.tripadvisor.es/data/graphql/batched':
response = json.loads(request.response.body)
for r in response:
if 'locations' in r['data']:
try:
for location in r['data']['locations']:
if 'reviewListPage' in location:
for review in location['reviewListPage']['reviews']:
r = Review(review['id'], review['userId'], review['title'], review['text'], review['language'])
reviews.append(r)
except Exception as e:
""
driver.quit()
return reviews
def initialize_driver(headless=True):
print("[.] Initialising chromedriver...")
chrome_options = Options()
if (headless):
chrome_options.add_argument("--headless")
chrome_options.add_argument("window-size=1920,1080")
chrome_options.add_argument("--log-level=3");
chrome_options.add_argument("--silent");
chrome_options.add_argument("--disable-logging")
# this allows selenium to accept cookies with a non-int64 'expiry' value
chrome_options.add_experimental_option("w3c", False)
# check OS to pick the correct driver
current_system = platform.system()
if (current_system == 'Windows'):
driver_path = os.path.join(path, "bin", "chromedriver.exe")
elif (current_system == 'Darwin'):
driver_path = os.path.join(path, "bin", "chromedriver")
else:
print('[!] Unsupported OS.')
sys.exit()
driver = webdriver.Chrome(
executable_path=driver_path,
service_log_path=os.path.join(path, "webdrive.log"),
# Don't verify self-signed cert, should help with 502 errors (https://github.com/wkeeling/selenium-wire/issues/55)
# seleniumwire_options={'verify_ssl': False},
options=chrome_options)
return driver;
def _authoriz_upd(self):
url = "https://app.zakupki360.ru/login"
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--headless')
with webdriver.Chrome(executable_path="./chromedriver",
options=chrome_options) as driver:
driver.implicitly_wait(10)
driver.get(url)
driver.find_element_by_id("mat-input-0").send_keys(
config.authoriz['login'])
driver.find_element_by_id("mat-input-1").send_keys(
config.authoriz['password'])
driver.find_element_by_tag_name('button').click()
driver.find_element_by_class_name("items")
resp = tuple({
req.headers['Authorization']
for req in driver.requests if req.response
})
try:
key = resp[0] if resp[0] else resp[1]
with open("auth_key", "w") as f_out:
f_out.write(key)
return key
except Exception as e:
print(resp)
print(e)
def __init__(self, *args, **kwargs):
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--headless')
chrome_options.add_argument('connection_keep_alive=False')
chrome_options.add_argument(
'disable-blink-features=AutomationControlled')
chrome_options.add_argument(
"user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:79.0) Gecko/20100101 Firefox/79.0"
)
options = {
'backend': 'mitmproxy',
'disable_encoding':
True # Tell the server not to compress the response
# Set the backend to mitmproxy
}
self.driver = webdriver.Chrome('chromedriver',
options=chrome_options,
seleniumwire_options=options)
self.driver.maximize_window()
self.driver.execute_cdp_cmd(
"Page.addScriptToEvaluateOnNewDocument", {
"source":
"""
Object.defineProperty(navigator, 'webdriver', {
get: () => undefined
})
"""
})
def _start_chrome_driver(self) -> webdriver:
"""Using Selenium start the google chrome browser headless.
All the browser requests and responses(har_fit data) will be recorded
using a BrowsermobProxy proxy server.
:return: Google chrome driver object.
"""
chrome_options = webdriver.ChromeOptions()
prefs = {"profile.default_content_setting_values.notifications": 2}
chrome_options.add_experimental_option("prefs", prefs)
chrome_options.set_capability(
'proxy', {
'httpProxy': f'{self.proxy.proxy}',
'noProxy': '',
'proxyType': 'manual',
'sslProxy': f'{self.proxy.proxy}'
})
# chrome_options.add_argument("--headless")
driver = webdriver.Chrome(chrome_options=chrome_options)
driver.set_page_load_timeout(999)
driver.delete_all_cookies()
return driver
def __init__(self, driver_path='/usr/local/bin/chromedriver', headless=True):
self.options = Options()
self.options.headless = headless
self.options.add_argument('--no-sandbox')
self.options.add_argument('--disable-dev-shm-usage')
self.driver = webdriver.Chrome(driver_path, chrome_options=self.options)
self.html = None
