def __init__(self, driver_name, driver_executable_path, driver_arguments,
browser_executable_path):
webdriver_base_path = f'selenium.webdriver.{driver_name}'
driver_klass_module = import_module(f'{webdriver_base_path}.webdriver')
driver_klass = getattr(driver_klass_module, 'WebDriver')
driver_options_module = import_module(f'{webdriver_base_path}.options')
driver_options_klass = getattr(driver_options_module, 'Options')
driver_options = driver_options_klass()
if browser_executable_path:
driver_options.binary_location = browser_executable_path
for argument in driver_arguments:
driver_options.add_argument(argument)
driver_kwargs = {
'executable_path': driver_executable_path,
f'{driver_name}_options': driver_options
}
driver_options.add_experimental_option(
"excludeSwitches", ["enable-automation"])
driver_options.add_experimental_option('useAutomationExtension', False)
self.driver = driver_klass(**driver_kwargs)
stealth(
self.driver,
vendor='Google Inc.',
platform='Win32',
webgl_vendor='Intel Inc.',
renderer="Intel Iris OpenGL Engine",
fix_hairline=True,
)
def session_create(config):
log.info("Creating session")
options = webdriver.ChromeOptions()
if config.get('headless', False) is True:
log.info("Headless mode")
options.add_argument("--headless")
if os.path.isfile("./chrome-win/chrome.exe"):
log.info("Found ./chrome-win/chrome.exe")
options.binary_location = "./chrome-win/chrome.exe"
driver = webdriver.Chrome(options=options)
stealth(
driver,
languages=["en-US", "en"],
vendor="Google Inc.",
platform="Win32",
webgl_vendor="Intel Inc.",
renderer="Intel Iris OpenGL Engine",
fix_hairline=True,
)
log.info("New session is: %s %s" %
(driver.session_id, driver.command_executor._url))
return driver
def initChromeDriver():
chrome_options = webdriver.ChromeOptions()
# prefs = {"download.default_directory" : "/some/path"}
# chromeOptions.add_experimental_option("prefs",prefs)
chrome_options.add_argument('--headless')
# chrome_options.add_argument('--no-sandbox')
# chrome_options.add_argument('--disable-dev-shm-usage')
chrome_options.add_argument('--incognito')
chrome_options.add_experimental_option("excludeSwitches",
["enable-automation"])
chrome_options.add_experimental_option('useAutomationExtension', False)
driver = webdriver.Chrome(os.environ.get('chromedriver'),
chrome_options=chrome_options)
# driver = webdriver.Chrome('./chromedriver',chrome_options=chrome_options)
stealth(
driver,
languages=["en-US", "en"],
vendor="Google Inc.",
platform="Win32",
webgl_vendor="Intel Inc.",
renderer="Intel Iris OpenGL Engine",
fix_hairline=True,
)
return driver
def initChromeDriver():
options = webdriver.ChromeOptions()
# options.add_argument('--headless')
#-- Uncomment the below two lines while running on MAC/LINUX --#
# options.add_argument('--no-sandbox')
# options.add_argument('--disable-dev-shm-usage')
options.add_argument('--incognito')
options.add_argument("--start-maximized")
options.add_argument('ignore-certificate-errors')
options.add_experimental_option("excludeSwitches", ["enable-automation"])
options.add_experimental_option('useAutomationExtension', False)
prefs = {"download.default_directory": f"{os.getcwd()}\\downloads\\"}
options.add_experimental_option("prefs", prefs)
#driver = webdriver.Chrome(os.environ.get('chromedriver'), options=options)
driver = webdriver.Chrome('./chromedriver', options=options)
stealth(
driver,
languages=["en-US", "en"],
vendor="Google Inc.",
platform="Win32",
webgl_vendor="Intel Inc.",
renderer="Intel Iris OpenGL Engine",
fix_hairline=True,
)
return driver
def browser_data():
options = webdriver.ChromeOptions()
options.add_argument("start-maximized")
options.add_argument("--headless")
# option.add_argument("--no-sandbox")
# options.add_argument("--disable-gpu")
# options.addArguments("--disable-dev-shm-usage")
options.add_experimental_option("excludeSwitches", ["enable-automation"])
options.add_experimental_option('useAutomationExtension', False)
driver = webdriver.Chrome(options=options)
stealth(
driver,
languages=["en-US", "en"],
vendor="Google Inc.",
platform="Win32",
webgl_vendor="Intel Inc.",
renderer="Intel Iris OpenGL Engine",
fix_hairline=True,
)
path = str(os.getcwd()).replace('\\', '/') + "/tests/static/test.html"
url = "https://bot.sannysoft.com/"
if os.name == 'nt':
url = 'file:///' + path
else:
url = 'file://' + path
print(url)
driver.get(url)
time.sleep(10)
metrics = driver.execute_cdp_cmd('Page.getLayoutMetrics', {})
width = math.ceil(metrics['contentSize']['width'])
height = math.ceil(metrics['contentSize']['height'])
screenOrientation = dict(angle=0, type='portraitPrimary')
driver.execute_cdp_cmd(
'Emulation.setDeviceMetricsOverride', {
'mobile': False,
'width': width,
'height': height,
'deviceScaleFactor': 1,
'screenOrientation': screenOrientation,
})
clip = dict(x=0, y=0, width=width, height=height, scale=1)
opt = {'format': 'png'}
if clip:
opt['clip'] = clip
result = driver.execute_cdp_cmd('Page.captureScreenshot', opt)
html = driver.page_source
driver.quit()
return html, result
def __init__(self, showWindow=True, proxy=None, downloadPath=None):
options = webdriver.ChromeOptions()
options.add_argument("--disable-dev-shm-usage")
options.add_argument("--no-sandbox")
options.add_argument('--disable-web-security')
options.add_argument('--allow-running-insecure-content')
options.add_argument("user-agent=DN")
options.add_experimental_option("excludeSwitches",
["enable-automation"])
options.add_experimental_option('useAutomationExtension', False)
if downloadPath is not None and isinstance(downloadPath, str):
absolute_path = os.path.abspath(downloadPath)
if not os.path.isdir(absolute_path):
raise FileNotFoundError(errno.ENOENT,
os.strerror(errno.ENOENT),
absolute_path)
options.add_experimental_option(
'prefs', {'download.default_directory': absolute_path})
if proxy is not None and isinstance(proxy, str):
options.add_argument("--proxy-server={}".format(proxy))
if not showWindow:
options.headless = True
self.driver = webdriver.Chrome(options=options)
self.Key = Keys
self.errors = []
stealth(
self.driver,
languages=["en-US", "en"],
vendor="Google Inc.",
platform="Win32",
webgl_vendor="Intel Inc.",
renderer="Intel Iris OpenGL Engine",
fix_hairline=True,
)
for function in [
'add_cookie', 'delete_all_cookies', 'delete_cookie',
'execute_script', 'execute_async_script', 'fullscreen_window',
'get_cookie', 'get_cookies', 'get_log',
'get_network_conditions', 'get_screenshot_as_base64',
'get_screenshot_as_file', 'get_screenshot_as_png',
'get_window_position', 'get_window_rect', 'get_window_size',
'maximize_window', 'minimize_window', 'implicitly_wait',
'quit', 'refresh', 'save_screenshot', 'set_network_conditions',
'set_page_load_timeout', 'set_script_timeout',
'set_window_position', 'set_window_rect', 'start_client',
'start_session', 'stop_client', 'switch_to_alert'
]:
setattr(self, function, getattr(self.driver, function))
def setup_browser(self):
stealth(
self.browser,
languages=["en-US", "en"],
vendor="Google Inc.",
platform="Win32",
webgl_vendor="Intel Inc.",
renderer="Intel Iris OpenGL Engine",
fix_hairline=True,
)
self.get_params(self.browser)
self.browser.execute_script(get_acrawler())
def setup_browser(self):
stealth(
self.browser,
languages=["en-US", "en"],
vendor="Google Inc.",
platform="Win32",
webgl_vendor="Intel Inc.",
renderer="Intel Iris OpenGL Engine",
fix_hairline=True,
)
self.get_params(self.browser)
# NOTE: Slower than playwright at loading this because playwright can ignore unneeded files.
self.browser.get("https://www.tiktok.com/@redbull")
self.browser.execute_script(get_acrawler())
self.browser.execute_script(get_tt_params_script())
def __init__(self):
options = webdriver.ChromeOptions()
options.add_argument("start-maximized")
#options.add_argument("--headless")
options.add_experimental_option("excludeSwitches",
["enable-automation"])
options.add_experimental_option('useAutomationExtension', False)
self.driver = webdriver.Chrome(options=options)
stealth(
self.driver,
languages=["en-US", "en"],
vendor="Google Inc.",
platform="Win32",
webgl_vendor="Intel Inc.",
renderer="Intel Iris OpenGL Engine",
fix_hairline=True,
)
def get_oauth2_token_stack(client):
try:
br = webdriver.Firefox(FirefoxDriverManager().install())
except Exception as e:
try:
options = webdriver.ChromeOptions()
options.add_argument("--headless")
options.add_argument("user-agent=DN")
options.add_experimental_option("excludeSwitches",
["enable-automation"])
options.add_experimental_option('useAutomationExtension', False)
br = webdriver.Chrome(ChromeDriverManager().install(),
options=options)
stealth(
br,
languages=["en-US", "en"],
vendor="Google Inc.",
platform="Win32",
webgl_vendor="Intel Inc.",
renderer="Intel Iris OpenGL Engine",
fix_hairline=True,
)
except Exception as e:
return None
#br.get('https://accounts.google.com/signin/oauth/identifier?client_id=717762328687-iludtf96g1hinl76e4lc1b9a82g457nn.apps.googleusercontent.com&as=JS6BM8cjL-8j9votansdkw&destination=https%3A%2F%2Fstackauth.com&approval_state=!ChRoYWVvLUlNMk5hSXJWUGlaSVl2WBIfc3lSa0lueENpb29lSU5vbEVpbVNxcUZGaGNkSEJoYw%E2%88%99AJDr988AAAAAXlBKc7PzEomxSzgNqd4wLptVlf0Ny3Qx&oauthgdpr=1&xsrfsig=ChkAeAh8T8JNDxCf2Zah5fb_rQ55OMiF8KmMEg5hcHByb3ZhbF9zdGF0ZRILZGVzdGluYXRpb24SBXNvYWN1Eg9vYXV0aHJpc2t5c2NvcGU&flowName=GeneralOAuthFlow')
br.get('https://stackoverflow.com/')
time.sleep(2)
br.find_element_by_xpath('/html/body/header/div/ol[2]/li[2]/a[1]').click()
time.sleep(0.5)
br.find_element_by_xpath(
'/html/body/div[3]/div[2]/div/div[2]/button[1]').click()
time.sleep(0.5)
br.find_element_by_id('Email').send_keys(client.userId)
#print(Fore.YELLOW + ' SENT EMAIL LOGIN ')
time.sleep(0.5)
br.find_element_by_xpath(
'/html/body/div/div[2]/div[2]/div[1]/form/div/div/input').click()
time.sleep(0.5)
br.find_element_by_id('password').send_keys(client.password)
#print(Fore.YELLOW + time_format() + ' SENT PASSWORD LOGIN ')
br.find_element_by_id('submit').click()
#print(Fore.BLUE + ' SUCCESSFULLY LOGGED IN ')
logged = True
def initChromeDriver():
options = webdriver.ChromeOptions()
options.add_argument("--log-level=3")
options.add_argument('--start-maximized')
options.add_argument('--headless')
options.add_argument('--incognito')
options.add_argument('ignore-certificate-errors')
options.add_experimental_option("excludeSwitches", ["enable-automation"])
options.add_experimental_option('useAutomationExtension', False)
driver = webdriver.Chrome(ChromeDriverManager().install(), options=options)
stealth(
driver,
languages=["en-US", "en"],
vendor="Google Inc.",
platform="Win32",
webgl_vendor="Intel Inc.",
renderer="Intel Iris OpenGL Engine",
fix_hairline=True,
)
return driver
def __init__(self, coords=None):
options = webdriver.ChromeOptions()
options.add_argument("start-maximized")
options.add_argument("start-maximized")
options.add_argument("--disable-extensions")
# options.add_argument("--headless")
options.add_experimental_option(
"prefs",
{
"profile.default_content_setting_values.geolocation": 1,
"profile.password_manager_enabled": False,
"credentials_enable_service": False
},
)
options.add_experimental_option("excludeSwitches",
["enable-automation"])
options.add_experimental_option("useAutomationExtension", False)
self.driver = webdriver.Chrome(options=options)
# Madison, WI Coords
#coords = {
# "latitude": 43.073051,
# "longitude": -89.40123,
# "accuracy": 98,
#}
if coords != None:
self.driver.execute_cdp_cmd("Page.setGeolocationOverride", coords)
stealth(
self.driver,
languages=["en-US", "en"],
vendor="Google Inc.",
platform="Win32",
webgl_vendor="Intel Inc.",
renderer="Intel Iris OpenGL Engine",
fix_hairline=True,
)
def initChromeDriver():
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--headless')
#-- Uncomment the below two lines while running on MAC/LINUX --#
# chrome_options.add_argument('--no-sandbox')
# chrome_options.add_argument('--disable-dev-shm-usage')
chrome_options.add_argument('--incognito')
chrome_options.add_experimental_option("excludeSwitches",
["enable-automation"])
chrome_options.add_experimental_option('useAutomationExtension', False)
# driver = webdriver.Chrome(os.environ.get('chromedriver'), chrome_options=chrome_options)
driver = webdriver.Chrome('./chromedriver', chrome_options=chrome_options)
stealth(
driver,
languages=["en-US", "en"],
vendor="Google Inc.",
platform="Win32",
webgl_vendor="Intel Inc.",
renderer="Intel Iris OpenGL Engine",
fix_hairline=True,
)
return driver
USERNAME = input("User Name : ")
PASSWORD = input("Password : "******"start-maximized")
options.add_experimental_option("excludeSwitches", ["enable-automation"])
options.add_experimental_option('useAutomationExtension', False)
options.add_experimental_option('excludeSwitches', ['enable-logging'])
# stealth 기능
stealth(
driver,
user_agent='DN',
languages=["en-US", "en"],
vendor="Google Inc.",
platform="Win32",
webgl_vendor="Intel Inc.",
renderer="Intel Iris OpenGL Engine",
fix_hairline=True,
) # Before Login, using stealth
login(USERNAME, PASSWORD)
stealth(
driver,
user_agent=
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.72 Safari/537.36',
languages=["en-US", "en"],
vendor="Google Inc.",
platform="Win32",
webgl_vendor="Intel Inc.",
import sys
from seleniumwire import webdriver
from selenium.webdriver.common.keys import Keys
from selenium_stealth import stealth
name = sys.argv[1]
email = sys.argv[2]
password = sys.argv[3]
options = webdriver.ChromeOptions()
options.add_argument("--disable-blink-features=AutomationControlled")
driver = webdriver.Chrome(options=options)
stealth(
driver,
languages=["en-US", "en"],
vendor="Google Inc.",
platform="Win32",
webgl_vendor="Intel Inc.",
renderer="Intel Iris OpenGL Engine",
fix_hairline=True,
)
driver.get(
"http://demo-rails-app-testing.ipm-corporation.com/signup?disallow_selenium=true"
)
element = driver.find_element_by_id("user_name")
element.send_keys(name)
element = driver.find_element_by_id("user_email")
element.send_keys(email)
element = driver.find_element_by_id("user_password")
element.send_keys(password)
element = driver.find_element_by_id("user_password_confirmation")
element.send_keys(password)
def browser(proxy):
global workers
# email = accountinfo.get_email()
# if not email:
# workers -= 1
# return
options = uc.ChromeOptions()
ua = UserAgent()
userAgent = ua.chrome
options.add_argument(f'user-agent={userAgent}')
#options.add_argument("--headless")
options.add_argument("--proxy-server={}://{}".format(proxyType, proxy))
options.add_argument("--disable-blink-features=AutomationControlled")
options.add_experimental_option("excludeSwitches", ["enable-automation"])
options.add_experimental_option('useAutomationExtension', False)
options.add_argument("--enable-features=ReaderMode")
options.add_argument('--ignore-ssl-errors=yes')
options.add_argument('--ignore-certificate-errors')
prefs1 = {
"profile.managed_default_content_settings.images": 1,
"profile.managed_default_content_settings.popups": 2,
"profile.default_content_setting_values.notifications": 2,
"profile.managed_default_content_settings.geolocation": 2,
"profile.managed_default_content_settings.stylesheets": 2,
}
options.add_experimental_option("prefs", prefs1)
options.binary_location = r"C:\Program Files (x86)\BraveSoftware\Brave-Browser\Application\brave.exe"
try:
driver = uc.Chrome(chrome_options=options)
stealth(
driver,
languages=["en-US", "en"],
user_agent=userAgent,
vendor="Google Inc.",
platform="Win32",
webgl_vendor="Intel Inc.",
renderer="Intel Iris OpenGL Engine",
fix_hairline=True,
)
url = "https://discord.com/register"
#url = "https://accounts.google.com/signin/v2/identifier?service=youtube&uilel=3&passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Faction_handle_signin%3Dtrue%26app%3Ddesktop%26hl%3Den%26next%3Dhttps%253A%252F%252Fwww.youtube.com%252F&hl=en&ec=65620&flowName=GlifWebSignIn&flowEntry=ServiceLogin"
try:
driver.get(url)
except WebDriverException:
workers -= 1
driver.quit()
return
Thread(target=await_tab, args=(driver, ), daemon=True).start()
sleeptime = 0.4
email = "testing1234crazytown{}@gmail.com".format(
random.randint(0, 100000))
username = "******".format(random.randint(0, 100000))
password = "******"
selection_month = random.choice([
'December', 'January', 'February', 'March', 'April', 'May', 'June',
'July', 'August', 'September', 'October', 'November'
])
selection_day = random.randint(1, 28)
try:
wait = WebDriverWait(driver, 30)
enter_searchbar = wait.until(
EC.presence_of_element_located((By.NAME, "email")))
slow_typing(enter_searchbar, email, 20)
time.sleep(sleeptime)
enter_user = driver.find_element_by_name("username")
slow_typing(enter_user, username, 20)
time.sleep(sleeptime)
enter_password = driver.find_element_by_name("password")
slow_typing(enter_password, password, 20)
time.sleep(sleeptime)
enter_month = driver.find_element_by_id("react-select-2-input")
slow_typing(enter_month, selection_month, 20)
enter_month.send_keys(Keys.ENTER)
time.sleep(sleeptime)
enter_day = driver.find_element_by_id("react-select-3-input")
slow_typing(enter_day, selection_day, 20)
enter_day.send_keys(Keys.ENTER)
time.sleep(sleeptime)
enter_year = driver.find_element_by_id("react-select-4-input")
slow_typing(enter_year, random.randint(1988, 1997), 20)
enter_year.send_keys(Keys.ENTER)
time.sleep(sleeptime)
try:
driver.find_element_by_xpath(
"//input[@type='checkbox']").click()
except Exception:
pass
driver.find_element_by_xpath("//button[@type='submit']").click()
try:
WebDriverWait(driver, 30).until(
EC.frame_to_be_available_and_switch_to_it(
(By.XPATH, '//iframe[contains(@title, "widget")]')))
WebDriverWait(driver, 10).until(
EC.element_to_be_clickable(
(By.CSS_SELECTOR, "div#checkbox.checkbox"))).click()
print("Found one!")
except Exception:
print("failed to find iframe!")
workers -= 1
driver.quit()
return
except Exception as e:
print(e)
except Exception as e:
print(e)
time.sleep(30)
workers -= 1
driver.quit()
return
def parse(self, response):
driver = response.meta['driver']
stealth(
driver,
languages=["en-US", "en"],
vendor="Google Inc.",
platform="Win32",
webgl_vendor="Intel Inc.",
renderer="Intel Iris OpenGL Engine",
fix_hairline=True,
)
driver.maximize_window()
time.sleep(3)
for _, val in self.df.iterrows():
if val['Id'] not in self.idList:
self.idList.append(val['Id'])
print(val['gSearchQuery'])
#input()
driver.get(
f"https://www.google.com/search?q={val['gSearchQuery']}")
#input()
time.sleep(1)
WebDriverWait(driver, 10).until(
EC.visibility_of_element_located(
(By.XPATH, "//div[@class='g']")))
#try:
# self.cntr += 1
# if self.cntr % 100 == 0:
# driver.refresh()
# time.sleep(random.randint(5,10))
# try:
# WebDriverWait(driver, 10).until(EC.visibility_of_element_located((By.XPATH, "//div[@class='g']")))
# except:
# input()
# SEARCHING FOR THE QUERY STRING #
# inputElem = driver.find_element_by_xpath("//input[@title='Search']")
# inputElem.clear()
# inputElem.send_keys(val['gSearchQuery'])
# serachBtnElem = driver.find_element_by_xpath("//button[@type='submit']")
# driver.execute_script("arguments[0].click()", serachBtnElem)
# time.sleep(1.5)
#WebDriverWait(driver, 5).until(EC.visibility_of_element_located((By.XPATH, "//div[@class='g']")))
# GETTING THE PAGE HEIGHT & SCROLLING GRADUALLY THROUGH THE PAGE TO LAOD ALL AVAILABLE LISTINGS #
# height = driver.execute_script("return document.body.scrollHeight")
# for i in range(1, (height//300)+1):
# driver.execute_script(f"window.scrollTo(0, {i*300});")
# time.sleep(0.1)
# driver.execute_script(f"window.scrollTo(0, -5000);")
html = driver.page_source
respObj = Selector(text=html)
website = respObj.xpath(
"//div[text()='Website']/parent::a/@href").get()
phone = respObj.xpath(
"normalize-space(//span[contains(@aria-label, 'phone number')]/text())"
).get()
if not website:
website = respObj.xpath(
"//div[text()='Website']/parent::div/parent::a/@href"
).get()
if not phone:
phone = respObj.xpath(
"normalize-space(//span[contains(@aria-label, 'Phone Number')]/text())"
).get()
if not phone:
phone = respObj.xpath(
"normalize-space((//div[text()='Website']/parent::div/parent::a/preceding-sibling::a)[1]//span[contains(@class,'details')]/div[2]/span[last()]/text())"
).get()
if not phone:
phone = respObj.xpath(
"normalize-space((//div[text()='Website']/parent::div/parent::a/preceding-sibling::a)[1]//span[contains(@class,'details')]/div[3]/span[last()]/text())"
).get()
if not phone:
phone = respObj.xpath(
"normalize-space(//div[text()='A']/parent::div/span/div[2]/span[last()]/text())"
).get()
yield {
'Id': val['Id'],
'City': val['State'],
'Area': val['City'],
'Restaurant Name': val['Restaurant name'],
'Average Rating': val['Average rating'],
'Number Of Reviews': val['Number of reviews'],
'Search Queries': val['gSearchQuery'],
'Website': self.extractUrl(website),
'Phone': phone
}
