def _authoriz_upd(self):
url = "https://app.zakupki360.ru/login"
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--headless')
with webdriver.Chrome(executable_path="./chromedriver",
options=chrome_options) as driver:
driver.implicitly_wait(10)
driver.get(url)
driver.find_element_by_id("mat-input-0").send_keys(
config.authoriz['login'])
driver.find_element_by_id("mat-input-1").send_keys(
config.authoriz['password'])
driver.find_element_by_tag_name('button').click()
driver.find_element_by_class_name("items")
resp = tuple({
req.headers['Authorization']
for req in driver.requests if req.response
})
try:
key = resp[0] if resp[0] else resp[1]
with open("auth_key", "w") as f_out:
f_out.write(key)
return key
except Exception as e:
print(resp)
print(e)
def __init__(self, *args, **kwargs):
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--headless')
chrome_options.add_argument('connection_keep_alive=False')
chrome_options.add_argument(
'disable-blink-features=AutomationControlled')
chrome_options.add_argument(
"user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:79.0) Gecko/20100101 Firefox/79.0"
)
options = {
'backend': 'mitmproxy',
'disable_encoding':
True # Tell the server not to compress the response
# Set the backend to mitmproxy
}
self.driver = webdriver.Chrome('chromedriver',
options=chrome_options,
seleniumwire_options=options)
self.driver.maximize_window()
self.driver.execute_cdp_cmd(
"Page.addScriptToEvaluateOnNewDocument", {
"source":
"""
Object.defineProperty(navigator, 'webdriver', {
get: () => undefined
})
"""
})
def scrape_kite_cookies(clientid, passwd, pin, headless=True):
base_url = 'https://kite.zerodha.com/'
demo_chart_url = 'https://kite.zerodha.com/chart/web/tvc/INDICES/NIFTY%2050/256265'
options = webdriver.ChromeOptions()
if headless:
options.add_argument('headless') # For headless mode
# endif
driver = webdriver.Chrome(chrome_options=options)
driver.get(base_url)
# Login using clientid and password
base_inps = driver.find_elements_by_tag_name('input')
base_inps[0].send_keys(clientid)
base_inps[1].send_keys(passwd)
base_button = driver.find_element_by_tag_name('button')
base_button.click()
time.sleep(2)
# Specify PIN
der_inp = driver.find_element_by_tag_name('input')
der_inp.send_keys(pin)
der_button = driver.find_element_by_tag_name('button')
der_button.click()
# Load a demo chart and sleep
driver.get(demo_chart_url)
time.sleep(2)
return driver.get_cookies()
def init_driver(self):
# settings selenium
driver_options = webdriver.ChromeOptions()
if self.browser_executable_path:
driver_options.binary_location = self.browser_executable_path
# # switch user_agent, proxy
# if hasattr(self, 'agent'):
# driver_options.add_argument(f"--user-agent='{next(self.agent)}'")
# if hasattr(self, 'proxy'):
# driver_options.add_argument(f"--proxy-server='{next(self.proxy)}'")
# add SELENIUM_DRIVER_ARGUMENTS options
for argument in self.driver_arguments:
driver_options.add_argument(argument)
# add experimental option: on/off image and notify
chrome_prefs = {"disk-cache-size": 4096}
if not self.driver_load_img:
chrome_prefs.update(
{
"profile.default_content_settings": {"images": 2},
"profile.managed_default_content_settings": {"images": 2}
}
)
if self.driver_disable_notify:
chrome_prefs.update(
{"profile.default_content_setting_values.notifications": 2}
)
driver_options.add_experimental_option("prefs", chrome_prefs)
self.driver = webdriver.Chrome(
executable_path=self.driver_executable_path,
options=driver_options
)
def __init__(self, location, page):
if location == "":
location = r"C:\Users\clone\Desktop"
options = webdriver.ChromeOptions()
options.add_experimental_option("excludeSwitches",
["enable-automation"])
options.add_argument('--disable-extensions')
options.add_argument("--incognito")
options.add_argument("--disable-plugins-discovery")
options.add_argument("--headless")
options.add_argument("--log-level=OFF")
self.parser = webdriver.Chrome(
executable_path=r"C:\Users\clone\Desktop\chromedriver.exe",
chrome_options=options)
self.page = self.parser.get(page)
name = ""
for letter in self.fileName():
if letter != ":" and letter != "\\":
name += letter
self.location = os.path.join(location, name +
".mp4") #Creates file name & location
time.sleep(5)
site = self.siteGetter()
header = self.headerGetter()
self.parser.quit()
self.properRipper(site, header, page, self.location)
def __init__(self, firstName, lastName, email, password):
self.firstName = firstName
self.lastName = lastName
self.email = email
self.password = password
#create a new webdriver with a header
options = webdriver.ChromeOptions()
options.add_experimental_option('excludeSwitches', ['enable-logging'])
options.add_argument('--ignore-certificate-errors')
options.add_argument('--incognito')
#remove headless option if you want to see the program work live
#options.add_argument('--headless')
options.add_argument(
'user-agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36"'
)
self.driver = webdriver.Chrome(ChromeDriverManager().install(),
options=options)
#used to manage header requests
self.driver.request_interceptor = interceptor
curURL = self.driver.current_url
self.driver.get("http://localhost:3000")
wait(self.driver, 15).until(EC.url_changes(curURL))
self.driver.maximize_window()
self.register()
self.login()
def get_info(link):
reviews = []
options = 1
driver = 1
try:
options = webdriver.ChromeOptions()
options.add_argument('--ignore-certificate-errors')
options.add_argument('--incognito')
options.add_argument('--headless')
driver = webdriver.Chrome(executable_path=binary_path, chrome_options=options)
driver.get(link)
wait = WebDriverWait(driver,120)
wait.until(wait_requests)
except TimeoutException:
print("Failed to load search bar at www.google.com")
except Exception as e:
print(e)
finally:
for request in driver.requests:
if request.response and request.response.headers['Content-Type'] == 'application/json' and request.path == 'https://www.tripadvisor.es/data/graphql/batched':
response = json.loads(request.response.body)
for r in response:
if 'locations' in r['data']:
try:
for location in r['data']['locations']:
if 'reviewListPage' in location:
for review in location['reviewListPage']['reviews']:
r = Review(review['id'], review['userId'], review['title'], review['text'], review['language'])
reviews.append(r)
except Exception as e:
""
driver.quit()
return reviews
def get_chrome_driver(driver_path):
"""
Instantiates a Chrome webdriver object
Args:
driver_path: path to the Chrome driver
Returns:
webdriver: returns a webdriver object
"""
options = webdriver.ChromeOptions()
options.page_load_strategy = 'normal'
options.add_argument("--disable-logging")
options.add_argument("--disable-login-animations")
options.add_argument("--disable-notifications")
options.add_argument("--disable-default-apps")
options.add_argument("--disable-extensions")
options.add_experimental_option("excludeSwitches",
["ignore-certificate-errors"])
options.add_argument("--disable-dev-shm-usage")
options.add_argument("--disable-browser-side-navigation")
options.add_argument("--headless")
options.add_argument("--hide-scrollbars")
options.add_argument('--log-level 3')
options.add_argument("--incognito")
options.add_argument("--no-zygote")
options.add_argument('--disable-gpu')
options.add_argument('--no-sandbox')
options.add_argument('--disable-infobars')
driver = webdriver.Chrome(driver_path, options=options)
driver.set_page_load_timeout(PAGE_LOAD_TIMEOUT_SEC)
return driver
def _start_chrome_driver(self) -> webdriver:
"""Using Selenium start the google chrome browser headless.
All the browser requests and responses(har_fit data) will be recorded
using a BrowsermobProxy proxy server.
:return: Google chrome driver object.
"""
chrome_options = webdriver.ChromeOptions()
prefs = {"profile.default_content_setting_values.notifications": 2}
chrome_options.add_experimental_option("prefs", prefs)
chrome_options.set_capability(
'proxy', {
'httpProxy': f'{self.proxy.proxy}',
'noProxy': '',
'proxyType': 'manual',
'sslProxy': f'{self.proxy.proxy}'
})
# chrome_options.add_argument("--headless")
driver = webdriver.Chrome(chrome_options=chrome_options)
driver.set_page_load_timeout(999)
driver.delete_all_cookies()
return driver
def __init__(self, email, username, password, proxy=None):
options = webdriver.ChromeOptions()
options.add_experimental_option("excludeSwitches", ["enable-logging"])
self.seleniumwire_options = {'proxy': {
'http': proxy,
'https': proxy
}
}
"""if proxy:
options.add_argument('--proxy-server=%s' % proxy)"""
self.platform = platform.system() #Support for linux system
"""if self.platform == 'Windows':
self.driver = webdriver.Chrome(options=options, executable_path=r"chromedriver.exe")
else:
self.driver = webdriver.Chrome(options=options, executable_path=r"chromedriver")"""
#Webdrivermanager install chromedriver binary as per system's platform
if proxy:
#check if proxy is working and also update timezone as per proxy to avoid detection
try:
self.timezone = requests.get('http://ip-api.com/json',proxies = self.seleniumwire_options['proxy'],timeout = 10).json()['timezone']
self.tz_params = {'timezoneId': self.timezone}
self.driver = webdriver.Chrome(options = options,executable_path = ChromeDriverManager().install(),seleniumwire_options=self.seleniumwire_options)
except:
self.timezone = None
free_print(f"{Fore.LIGHTMAGENTA_EX}[!]{Style.RESET_ALL} " + 'Skipping Proxy as Connection Issue Have you entered it in the correct format?')
self.driver = webdriver.Chrome(options = options,executable_path = ChromeDriverManager().install())
else:
self.timezone = None
self.driver = webdriver.Chrome(options = options,executable_path = ChromeDriverManager().install())
self.email= email
self.username = username
self.password = password
def options(self):
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--disable-dev-shm-usage')
print('chrome options loaded')
return chrome_options
def build(cfg, fetch_driver=True):
"""
builds a selenium-webdriver object with the specified configuration
:param cfg: Configuration object
:param fetch_driver: bool (default=True) fetches driver binaries
:return: selenium-wire Webdriver object
"""
if cfg.proxy is not None:
options = cfg.proxy.create_options()
else:
options = {}
if cfg.driver in WebDriver.FIREFOX_DRIVER_NAMES:
d = webdriver.Firefox
o = webdriver.FirefoxOptions()
if cfg.profile is None:
p = webdriver.FirefoxProfile()
else:
p = webdriver.FirefoxProfile(cfg.profile)
p.set_preference("general.useragent.override", cfg.user_agent)
p.set_preference("media.volume_scale", "0.0")
'''
if cfg.proxy is not None:
p = cfg.proxy.update_preferences(p)
'''
elif cfg.driver in WebDriver.CHROME_DRIVER_NAMES:
d = webdriver.Chrome
o = webdriver.ChromeOptions()
o.add_argument("user-agent={0}".format(cfg.user_agent))
if cfg.user_data_dir:
o.add_argument("user-data-dir={0}".format(cfg.user_data_dir))
'''
if cfg.proxy is not None:
o.add_argument("--proxy-server={0}".format(cfg.proxy.for_chrome()))
'''
p = None
else:
raise NotImplementedError
if fetch_driver:
Loader.fetch(cfg.executable_path, cfg.debug, cfg.driver)
o.binary_location = cfg.executable_path
o.headless = cfg.headless
if cfg.driver in WebDriver.FIREFOX_DRIVER_NAMES:
if cfg.proxy is None:
return d(p, cfg.binary, options=o)
else:
'''return d(p, cfg.binary, options=o, proxy=cfg.proxy, seleniumwire_options=options)'''
return d(p,
cfg.binary,
options=o,
seleniumwire_options=options)
elif cfg.driver in WebDriver.CHROME_DRIVER_NAMES:
if cfg.proxy is None:
return d(options=o)
else:
'''return d(options=o, proxy=cfg.proxy, seleniumwire_options=options)'''
return d(options=o, seleniumwire_options=options)
def __buildDriver__(self, driver_options):
# Ubuntu
if (os.name == 'posix'):
chromedriver = 'chromedriver'
# Windows
if (os.name == 'nt'):
chromedriver = 'chromedriver.exe'
if chromedriver:
if config.DRIVER_NAME == 'Chrome':
logger.info("Using Chrome Driver ...")
options = webdriver.ChromeOptions()
options.add_experimental_option("excludeSwitches",
["enable-automation"])
options.add_experimental_option('useAutomationExtension',
False)
options.add_experimental_option('w3c', False)
for driver_option in driver_options:
# if "--proxy-server" in driver_option:
# print(f"\nADDING PROXY: [{driver_option}]\n")
options.add_argument(driver_option)
capabilities = None
if config.PROXY:
# print(f"\nADDING PROXY: [{config.PROXY}]\n")
# prox = Proxy()
# prox.proxy_type = ProxyType.MANUAL
# prox.http_proxy = config.PROXY
# # prox.socks_proxy = config.PROXY
# prox.ssl_proxy = config.PROXY
capabilities = webdriver.DesiredCapabilities.CHROME
capabilities['loggingPrefs'] = {'performance': 'ALL'}
# prox.add_to_capabilities(capabilities)
if capabilities:
self.driver = webdriver.Chrome(
desired_capabilities=capabilities, options=options)
# self.driver = config.DRIVER_NAME(desired_capabilities=capabilities, options=options)
else:
self.driver = webdriver.Chrome(chromedriver,
options=options)
# self.driver = config.DRIVER_NAME(chromedriver, options=options)
else:
logger.info("Using Firefox Driver ...")
self.driver = webdriver.Firefox()
self.min_wait = WebDriverWait(self.driver, 5)
self.max_wait = WebDriverWait(self.driver, 20)
self.els_css = self.driver.find_elements_by_css_selector
self.el_css = self.driver.find_element_by_css_selector
self.els_xpath = self.driver.find_elements_by_xpath
self.el_xpath = self.driver.find_element_by_xpath
self.driver.set_script_timeout(30)
return self.driver
def test_wait_for_request_headless_chrome(self):
# https://github.com/wkeeling/selenium-wire/issues/6
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument("--headless")
def get_page_response(url):
# check for current os
if os.name == 'posix':
# osx
driver_path = '/usr/local/bin/chromedriver'
elif os.name == 'nt':
# win32
driver_path = 'C:\chromedriver\chromedriver'
else:
print('Unknown operating system!!!')
exit()
driver = webdriver.Chrome(chrome_options=chrome_options,
executable_path=driver_path)
driver.get(url)
request = driver.wait_for_request(url, timeout=3)
print(request)
self.assertEqual(request.path, 'https://www.google.com/')
get_page_response('https://www.google.com')
def load_browser():
global driver
global ublock_guid
chrome_options = webdriver.ChromeOptions()
# chrome_options.binary_location = "/applications/developer/google\ chrome.app/Contents/MacOS/Google\ Chrome"
# chrome_options.add_argument("--disable-web-security") # messes up ublock
# chrome_options.add_argument('--headless')
chrome_options.add_extension('ubo_1_30_4_0.crx')
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--disable-gpu')
chrome_options.add_argument("--ignore-certificate-errors")
chrome_options.add_argument("--enable-javascript")
chrome_options.add_argument("--disable-chrome-google-url-tracking-client")
chrome_options.add_argument("--safebrowsing-disable-download-protection")
chrome_options.add_argument("--disable-domain-reliability")
chrome_options.add_argument("--allow-running-insecure-content")
chrome_options.add_argument(
"--unsafely-treat-insecure-origin-as-secure=http://host:port")
chrome_options.add_argument(
'--user-agent=Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.92 Safari/537.36'
)
driver = webdriver.Chrome(executable_path="./chromedriver",
options=chrome_options)
driver.header_overrides = {'Referer': 'com.google.android.gm'}
# display = Display(visible=0, backend="xephyr", size=(800, 600))
# display.start()
extension_uri = "chrome-extension://cjpalhdlnbpafiamejdnhcphjbkeiagm/logger-ui.html?popup=1#_"
driver.implicitly_wait(page_delay)
driver.get(extension_uri)
ublock_guid = driver.current_window_handle
def setup_browser(self):
# Start Headless Display
try:
self.display = Display(visible=False, size=(self.width, self.height))
self.display.start()
except EasyProcessError as e:
self.headless: bool = True
# Setup Chrome/Chromium - https://peter.sh/experiments/chromium-command-line-switches/
self.options = webdriver.ChromeOptions()
self.options.add_argument('--disable-blink-features=AutomationControlled') # Disable WebDriver Detection
self.options.add_argument(f'--user-agent={self.user_agent}') # To Identify Bot To Server Owners
# Disable Code That Might Take CPU Cycles
self.options.add_argument('--disable-extensions') # Chrome Apps/Extensions
# self.options.add_argument('--disable-plugins') # No Longer Exists (Used To Be For Flash/Java/etc...)
self.options.add_argument('--disable-notifications') # Disable Website Notification Capabilities
self.options.add_argument('--disable-sync') # Disable Syncing Chrome Account Data
# Force Lite Mode Pages
# self.options.add_argument('--force-enable-lite-pages') # Force Lite Mode Pages
# To Disable Accidental Audio Playing
self.options.add_argument('--mute-audio') # Disables Audio Playing
# self.options.add_argument('--incognito') # Incognito Mode
# self.options.add_argument('--no-referrers') # Hide Referral Websites
# self.options.add_argument('--no-experiments') # Disable Chrome Flags
# self.options.add_argument('--remote-debugging-port=9222') # For Debugging
if self.headless:
# print("No V-Screen Available!!! Starting Headless!!!")
self.options.add_argument('--headless')
def browsSart(self):
try:
uio = [
'https://www.junno.co/news',
'https://www.junno.co/famouspeople',
'https://www.junno.co/lyrics', 'https://www.junno.co/shows/3',
'https://www.junno.co/movies/1'
]
ip = self.rad_c()
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument("--headless")
chrome_options.binary_location = os.environ.get(
"GOOGLE_CHROME_BIN")
chrome_options.add_argument("--disable-dev-shm-usage")
chrome_options.add_argument(
'window-size=1920,{m}'.format(m=random.randint(708, 1080)))
chrome_options.add_argument("--no-sandbox")
self.driver = webdriver.Chrome(
executable_path=os.environ.get("CHROMEDRIVER_PATH"),
chrome_options=chrome_options)
ty = self.myip()
print(ty)
self.driver._client.set_header_overrides(
headers={
'X-Forwarded-For':
'{n}'.format(n=ip),
'Via':
'{n}'.format(n=ip),
'X-Real-IP':
'{n}'.format(n=ip),
'True-Client-IP':
'{n}'.format(n=ip),
'Forwarded':
"'for={n}:{m}'".format(n=ip,
m=random.randint(20000, 60000)),
'User-Agent':
'{n}'.format(n=random.choice(self.n))
})
self.driver.get(uio[random.randint(1, len(uio) - 1)])
element_present = EC.presence_of_element_located(
(By.XPATH, '//body'))
WebDriverWait(self.driver, 5).until(element_present)
self.driver.find_element_by_id('{n}'.format(n=ty)).click()
element_present = EC.presence_of_element_located(
(By.XPATH, '//body'))
data = WebDriverWait(self.driver, 5).until(element_present)
time.sleep(1)
self.driver.delete_all_cookies()
self.driver.execute_script('window.localStorage.clear();')
self.driver.execute_script('window.sessionStorage.clear();')
self.driver.quit()
print('good')
except Exception:
print('bad error CheckProxy')
if self.driver:
self.driver.quit()
def __init__(self, settings):
self.settings = settings
self.fakemail = FakeMail()
self.chrome_options = webdriver.ChromeOptions()
self.chrome_options.add_experimental_option('excludeSwitches',
['enable-logging'])
def __init__(self, proxy_class, price_class):
"""Создаем экземпляр бота"""
Thread.__init__(self)
if settings.DEBUG:
print('CREATE BOT')
self.proxies = proxy_class
self.price = price_class
self.no_proxy = False
self.use_proxy = 1
if self.use_proxy == 2:
self.proxy = ''
else:
self.proxy, clean_proxy = self.proxies.get_new_proxy()
"""Подключаем настройки прокси к экземпляру бота"""
options = {'proxy': self.proxy}
wind_width = random.randint(1000, 1920)
wind_height = random.randint(750, 1080)
with open(
f'{settings.MEDIA_DIR}/user-agents/user_agents_for_chrome_pk.txt',
'r',
encoding='utf-8') as f:
user_agent = random.choice([line.strip() for line in f])
self.options = webdriver.ChromeOptions()
self.options.add_argument("ignore-certificate-errors")
self.options.add_argument(f'window-size={wind_width},{wind_height}')
self.options.add_argument("--disable-blink-features")
self.options.add_argument(
"--disable-blink-features=AutomationControlled")
self.options.add_argument("--headless")
self.options.add_argument('--disable-extensions')
self.options.add_argument('--profile-directory=Default')
"""Добавялем юзер агент из списка"""
self.options.add_argument(f"user-agent={user_agent}")
"""Создаем экземпляр браузера"""
if self.use_proxy == 1:
self.driver = webdriver.Chrome(
options=self.options,
executable_path=f'{settings.MEDIA_DIR}/drivers/chromedriver',
seleniumwire_options=options,
)
else:
self.driver = webdriver.Chrome(
options=self.options,
executable_path=f'{settings.MEDIA_DIR}/drivers/chromedriver',
)
"""Создаем необходимые переменные для работы"""
self.reboot = False
self.search_status = False
self.step = 0
self.target = []
def get_chrome():
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('headless')
chrome_options.add_argument("--mute-audio")
chrome_options.add_experimental_option("excludeSwitches",
["enable-logging"])
return webdriver.Chrome(ChromeDriverManager().install(),
chrome_options=chrome_options)
def launch_browser(headers=None,
user_agent=None,
proxy=None,
browser_type="Firefox"):
options = {}
if proxy:
proxy = {
"http": proxy,
"https": proxy,
}
options["proxy"] = proxy
if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"):
directory = sys._MEIPASS
else:
directory = os.path.dirname(__file__)
driver = None
if browser_type == "Firefox":
matches = ["geckodriver.exe", "geckodriver"]
driver_paths = list(
map(lambda match: os.path.join(directory, match), matches))
found_paths = [
driver_path for driver_path in driver_paths
if os.path.exists(driver_path)
]
if found_paths:
driver_path = found_paths[0]
opts = webdriver.FirefoxOptions()
# opts.add_argument("--headless")
profile = webdriver.FirefoxProfile()
if not user_agent:
user_agent = generate_user_agent()
profile.set_preference("general.useragent.override", user_agent)
driver = webdriver.Firefox(
firefox_profile=profile,
executable_path=driver_path,
options=opts,
seleniumwire_options=options,
)
else:
message = f"Download geckodriver from https://github.com/mozilla/geckodriver/releases/tag/v0.27.0 and paste it in {directory}"
input(message)
else:
driver_path = os.path.join(directory, "chromedriver.exe")
opts = webdriver.ChromeOptions()
opts.add_argument(f"--proxy-server={opts}")
driver = webdriver.Chrome(executable_path=driver_path,
options=opts,
seleniumwire_options=options)
if not driver:
input("DRIVER NOT FOUND")
exit(0)
driver.set_window_size(1920, 1080)
browser = driver
if headers:
browser._client.set_header_overrides(headers=headers)
return browser
def startBrowser(self):
profile = webdriver.ChromeOptions()
profile.add_argument('--no-sandbox')
profile.add_experimental_option('excludeSwitches',
['enable-automation'])
if self.headless: profile.add_argument('--headless')
self.browser = webdriver.Chrome(executable_path=self.chromedriver,
options=profile)
self.browser.set_page_load_timeout(180)
self.browser.set_window_size(1800, 1020)
def initBrowser(browserConfig, isUseSeleniumwire=False):
"""Init browser driver for selenium
Args:
browserConfig (dict): browser config
isUseSeleniumwire (bool): get selenium-wire webdriver or not. Default is False.
Returns:
driver
Raises:
Examples:
browserConfig
normal Selenium: {'headless': True, 'pageLoadTimeout': 10}
Selenium-wire: {'headless': True, 'pageLoadTimeout': 10, "seleniumwire": {"disable_capture":True}}
"""
webdriver = getWebDirver(isUseSeleniumwire=isUseSeleniumwire)
chromeOptions = webdriver.ChromeOptions()
if "headless" in browserConfig:
isCurHeadless = browserConfig["headless"]
if isCurHeadless:
chromeOptions.add_argument('--headless')
if "disableGpu" in browserConfig:
isDisableGpu = browserConfig["disableGpu"]
if isDisableGpu:
chromeOptions.add_argument('--disable-gpu')
# for debug
# chromeOptions.add_experimental_option("excludeSwitches", ["enable-automation"])
# chromeOptions.add_experimental_option('useAutomationExtension', False)
# chromeOptions.add_argument('window-size=1920x1080')
# chromeOptions.add_argument('--window-size=1920x1080')
driverConfigDict = {"options": chromeOptions}
if "seleniumwire" in browserConfig:
if isUseSeleniumwire:
driverConfigDict["seleniumwire_options"] = browserConfig[
"seleniumwire"]
else:
del browserConfig["seleniumwire"]
driver = webdriver.Chrome(**driverConfigDict)
# driver = webdriver.Chrome(options=chromeOptions)
# # for debug
# driver.set_window_size(1920, 1080)
if "pageLoadTimeout" in browserConfig:
curPageLoadTimeout = browserConfig["pageLoadTimeout"]
driver.set_page_load_timeout(curPageLoadTimeout)
return driver
def initialize_driver():
option = webdriver.ChromeOptions()
chrome_prefs = {}
option.experimental_options["prefs"] = chrome_prefs
chrome_prefs["profile.default_content_settings"] = {"images": 2}
chrome_prefs["profile.managed_default_content_settings"] = {"images": 2}
driver = webdriver.Chrome(options=option, seleniumwire_options=options)
# driver.get("https://ipv4.icanhazip.com/")
# time.sleep(2)
# driver.get("https://ipv4.icanhazip.com/")
# print("Proxy: ", driver.find_element_by_tag_name("body").text)
return driver
def __init__(self):
self.base_url = "http://moneypuck.com"
opts = webdriver.ChromeOptions()
sw_opts = {"disable_encoding": True}
opts.add_argument("--headless")
self.driver = webdriver.Chrome(options=opts,
seleniumwire_options=sw_opts)
with open("nhl_teams.yaml", "r") as f:
self.teams = yaml.load(f, Loader=yaml.FullLoader)
self.teams_inv = {v: k for k, v in self.teams.items()}
def create_headers(self,email,password):
options = webdriver.ChromeOptions()
# options.add_argument("--headless")
with webdriver.Chrome(executable_path=ChromeDriverManager().install(),options=options) as driver:
_cookie,_ua = self.login_to_fb(driver,email,password)
print(_cookie)
print(_ua)
self.headers['cookie'] = _cookie
self.headers['user-agent'] = _ua
filename = email[:email.index('@')]
export_file_path = self.headers_path + filename + '.json'
with open(export_file_path,'w') as outputfile:
json.dump(self.headers, outputfile,indent=4)
def get_driver(name: str, path: Optional[str]) -> Driver:
driver_class = DRIVER_NAME_TO_CLASS[name]
chrome_options = webdriver.ChromeOptions()
chrome_options.headless = True
args = ({
"executable_path": path,
"chrome_options": chrome_options
} if path else {
"chrome_options": chrome_options
})
return driver_class(**args)
def login(account_data: dict, initialization_headers: object,
initialization_cookies: object, requests_map: dict):
headers = dict()
cookies = {}
def interceptor(request):
try:
headers["x-ig-app-id"] = request.headers["X-IG-App-ID"]
new = request.headers["Cookie"].split(';')
for new_one in new:
cooka = new_one.split('=')
cookies[cooka[0].strip()] = cooka[1].strip()
except AttributeError as errors_inter:
logger.warning(
f"The authorization process was not correct.!!! Error - {errors_inter}"
)
sys.stdout.write(f"Error {errors_inter}")
try:
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument("--window-size=10x10")
driver = webdriver.Chrome(executable_path=config.PATH_TO_CHROMEDRIVER,
options=chrome_options)
driver.get(requests_map["main_url"])
time.sleep(5)
username_field = driver.find_element_by_name(
requests_map["login"]["username_field"])
password_field = driver.find_element_by_name(
requests_map["login"]["password_field"])
submit_elem = driver.find_element_by_tag_name('button')
username_field.send_keys(account_data["username"])
password_field.send_keys(account_data["password"])
submit_elem.click()
driver.request_interceptor = interceptor
request = driver.wait_for_request(requests_map["login"]["uri"])
headers["x-ig-www-claim"] = request.response.headers[
"x-ig-set-www-claim"]
driver.close()
set_cookies(initialization_cookies, cookies)
if set_headers(initialization_headers, headers, cookies):
return {"status": True}
return {"status": False}
except Exception as errors:
logger.warning(
f"The authorization process was not correct.!!! Error - {errors}")
return {"status": False}
def __init__(self, username, password, brand, is_main):
self.username = username
self.password = password
self.brand = brand
self.is_main = is_main
self.previousComments = open("Instagram_Comment_Names", "a")
self.accounts_already_commented_on = open("Instagram_Comment_Names",
"r").read().split(" ")
self.comment_filter = {
"post_min_number_of_comments": 0,
"post_max_number_of_comments": 10000
}
self.comment_pool = []
self.comments_done = 0
self.num_comments = 1
self.could_not_comment = False
self.could_not_comment_popup = False
self.is_liking = 1
self.liking_blocked = False
options = webdriver.ChromeOptions()
#chrome_options.headless = True
#chrome_options.add_argument("--headless") # Runs Chrome in headless mode.
#chrome_options.add_argument('--no-sandbox') # Bypass OS security model
#chrome_options.add_argument('--disable-gpu') # applicable to windows os only
#chrome_options.add_argument('start-maximized') #
#chrome_options.add_argument('disable-infobars')
#chrome_options.add_argument("--disable-extensions")
#chrome_options.add_argument('--proxy-server=%s' % PROXY)
user_agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36'
options.add_argument(f'user-agent={user_agent}')
#options.binary_location = os.getcwd() + "/headless-chromium"
self.driver = webdriver.Chrome(
executable_path=os.getcwd() + "/chromedriver",
chrome_options=options,
seleniumwire_options={
'request_storage_base_dir': '/tmp',
# 'proxy': {
# 'https': f'https://*****:*****@{PROXY}'
# }
})
#self.driver.get("https://nordvpn.com/what-is-my-ip/")
#print(self.driver.find_element_by_css_selector("body > div.Page.overflow-hidden.d-flex.flex-column > div:nth-child(1) > div > div > div > div > div > div > div.col-xs-12.col-sm-6.mb-7.mb-sm-0 > h3.Title.mb-6.js-ipdata-ip-address").text)
self.inception_time = time.time()
print("Inception:", self.inception_time)
if not self.login():
sys.exit("Login failed")
def _get_default_driver(self, headless):
seleniumwire_options = {
'connection_keep_alive': False,
'connection_timeout': 20
}
options = webdriver.ChromeOptions()
options.add_argument('window-size=1920,1080')
if headless is True:
options.add_argument('--disable-extensions')
options.add_argument('--headless')
options.add_argument('--disable-gpu')
options.add_argument('--no-sandbox')
return webdriver.Chrome(options=options,
seleniumwire_options=seleniumwire_options)
