def main(args):
print(
"-------------------------------------------------------------------------------------------------------------------------------------\n"
"███████╗ ██████╗ ████████╗ ██████╗ ██████╗ █████╗ ███████╗ █████╗ ███████╗ ██████╗██████╗ █████╗ ██████╗ ██╗███╗ ██╗ ██████╗\n"
+
"██╔════╝██╔═══██╗╚══██╔══╝██╔═══██╗██╔════╝██╔══██╗██╔════╝██╔══██╗ ██╔════╝██╔════╝██╔══██╗██╔══██╗██╔══██╗██║████╗ ██║██╔════╝\n"
+
"█████╗ ██║ ██║ ██║ ██║ ██║██║ ███████║███████╗███████║ ███████╗██║ ██████╔╝███████║██████╔╝██║██╔██╗ ██║██║ ███╗\n"
+
"██╔══╝ ██║ ██║ ██║ ██║ ██║██║ ██╔══██║╚════██║██╔══██║ ╚════██║██║ ██╔══██╗██╔══██║██╔═══╝ ██║██║╚██╗██║██║ ██║\n"
+
"██║ ╚██████╔╝ ██║ ╚██████╔╝╚██████╗██║ ██║███████║██║ ██║ ███████║╚██████╗██║ ██║██║ ██║██║ ██║██║ ╚████║╚██████╔╝\n"
+
"╚═╝ ╚═════╝ ╚═╝ ╚═════╝ ╚═════╝╚═╝ ╚═╝╚══════╝╚═╝ ╚═╝ ╚══════╝ ╚═════╝╚═╝ ╚═╝╚═╝ ╚═╝╚═╝ ╚═╝╚═╝ ╚═══╝ ╚═════╝\n"
"-------------------------------------------------------------------------------------------------------------------------------------\n"
)
# Global variables
chromedriver_autoinstaller.install()
global driver
driver = webdriver.Chrome()
global fotocasa_base_path
fotocasa_base_path = "https://www.fotocasa.es"
url = initVariables(args)
linksList = obtainLinks(url)
data_homes = obtainDataHomes(linksList)
driver.close()
writeDataToFile(data_homes)
time.sleep(2)
def web_page_loader(url):
# Works, when browser open and logged in
# Next few lines are for checking if the webdriver is there and what version.
# This is needed for web_page_loader and for login_and_get_page functions
chromedriver_autoinstaller.install(
) # Check if the current version of chromedriver exists
# and if it doesn't exist, download it automatically, then add chromedriver to path
driver = webdriver.Chrome()
# driver = webdriver.Chrome(ChromeDriverManager().install()) # always installs
if "hs." not in url: # if this url is not from Hessari
print(
"Are you sure you gave the right link for HS ruoka page? Try again."
)
exit()
try: # if this is not a URL
html = request.urlopen(url).read().decode('utf8')
except:
print("The URL give could not be opened")
exit()
soup = BeautifulSoup(html, 'html.parser') # contains links to recipes
soup_lxml = bs4.BeautifulSoup(html,
'lxml') # (does not give links to recipes)
sleep(3) # if you want to wait 3 seconds for the page to load
return soup, soup_lxml
def driver_start(self, is_headless=False):
'''
Start driver
구글 드라이버 실행
Parameter
-----------------------------
start_date: 지정한 날짜부터 크롤링 시작
'''
version = chromedriver_autoinstaller.get_chrome_version()[:2]
if is_headless:
webdriver_options = webdriver.ChromeOptions()
webdriver_options.add_argument('headless')
try:
self.driver = webdriver.Chrome(
'C:\\Users\\Chan\\.conda\\envs\\baseball\\lib\\site-packages\\chromedriver_autoinstaller\\'
+ version + '\\chromedriver',
options=webdriver_options)
except:
chromedriver_autoinstaller.install()
else:
try:
self.driver = webdriver.Chrome(
'C:\\Users\\Chan\\.conda\\envs\\baseball\\lib\\site-packages\\chromedriver_autoinstaller\\'
+ version + '\\chromedriver')
except:
chromedriver_autoinstaller.install()
def get_authorization_code(client_id, username, password):
"""
Uses Selenium to log in and get the Webex authorization code from the URL
"""
url = BASE_URL + '/authorize' + f'?response_type=code&client_id={client_id}&redirect_uri={uri}&scope=meeting%3Arecordings_read%20spark%3Akms%20meeting%3Arecordings_write&state=whj'
try:
options = Options()
options.add_argument("--headless")
chromedriver_autoinstaller.install()
driver = webdriver.Chrome(options=options)
driver.implicitly_wait(30)
driver.get(url)
driver.find_element_by_name("IDToken1").send_keys(username)
driver.find_element_by_id("IDButton2").click()
driver.find_element_by_name("IDToken2").send_keys(password)
driver.find_element_by_id("Button1").click()
url_code = driver.current_url
code = url_code.split('?')[1].split('&')[0].split('=')[1]
except Exception as err:
driver.save_screenshot("get_authorization_code_error.png")
raise Exception(
f"ERROR Getting WebEx authorization code: {err.__class__.__name__}: {str(err)}"
)
finally:
driver.close()
return code
def browser_login(login_url,
user_css_selector,
user,
password_css_selector,
password,
butt_css_selector,
pop_css_selector=None):
chromedriver_autoinstaller.install()
options = webdriver.ChromeOptions()
options.add_argument('--headless')
with webdriver.Chrome(options=options) as driver:
# Login
driver.get(login_url)
if pop_css_selector:
pop_up_butt = driver.find_element_by_css_selector(pop_css_selector)
pop_up_butt.click()
user_input = driver.find_element_by_css_selector(user_css_selector)
user_input.send_keys(user)
pass_input = driver.find_element_by_css_selector(password_css_selector)
pass_input.send_keys(password)
login_butt = driver.find_element_by_css_selector(butt_css_selector)
login_butt.click()
return driver.get_cookies()
def restart_freebox(password,
url='http://mafreebox.freebox.fr/login.php',
headless=True,
test=False):
chrome_options = Options()
chrome_options.add_argument("--window-size=1920,1280")
if headless:
chrome_options.add_argument('headless')
chromedriver_autoinstaller.install()
driver = webdriver.Chrome(options=chrome_options)
try:
driver.get(url)
WebDriverWait(driver, 30).until(
EC.element_to_be_clickable(
(By.NAME, "password_"))).send_keys(password + Keys.RETURN)
WebDriverWait(driver, 30).until(
EC.element_to_be_clickable(
(By.XPATH, "//a[@aria-label='Menu démarrer']"))).click()
WebDriverWait(driver, 30).until(
EC.element_to_be_clickable(
(By.XPATH, "//*[contains(text(), 'Redémarrer')]"))).click()
WebDriverWait(driver, 30).until(
EC.element_to_be_clickable(
(By.ID, f'button-{1006 + test}'))).click()
print('Freebox Restarted')
finally:
driver.quit()
def get_bb_slot(url):
# auto-install chromedriver
chromedriver_autoinstaller.install()
driver = webdriver.Chrome()
driver.get(url)
print("Please login using OTP and then wait for a while.")
time.sleep(60)
while 1:
driver.get(url)
time.sleep(2)
print("Trying to find a slot!")
try:
driver.find_element_by_xpath("//button[@id = 'checkout']").click()
time.sleep(5) #driver take a few sec to update the new url
src = driver.page_source
if "checkout" in driver.current_url and not "Unfortunately, we do not have" in src:
print("Found the slots!")
for i in range(60):
notify("Slots Available!",
"Please go and choose the slots!")
time.sleep(20)
except Exception as e:
print(
"If this message pops up multiple times, please find the error and create a PR!"
)
print(e)
pass
print("No Slots found. Will retry again.")
time.sleep(50)
def __init__(self, *args, **kwargs):
self.myurls = kwargs.get('myurls', [])
chromedriver_autoinstaller.install()
chrome_options = Options()
chrome_options.add_argument("--headless")
self.driver = webdriver.Chrome(options=chrome_options)
super(GoFileSpider, self).__init__(*args, **kwargs)
def init():
#Chrome driver updates
cd_ai.install()
#Folders Init
FolderInit(v.ProfilePath)
FolderInit(v.ExtensionPath)
#Chrome Options
capa = DesiredCapabilities.CHROME
capa["pageLoadStrategy"] = "none"
chrome_options = Options()
chrome_options.add_argument("--user-data-dir=" +
os.path.abspath(v.ProfilePath))
#Brave Browser Support
if v.Brave: chrome_options.binary_location = v.Brave
#CRX loads
for fn in [
f for f in os.listdir(v.ExtensionPath)
if os.path.isfile(os.path.join(v.ExtensionPath, f))
]:
chrome_options.add_extension(v.ExtensionPath + fn)
v.browser = webdriver.Chrome(options=chrome_options,
desired_capabilities=capa)
#Init Web Driver Wait
v.elementWait = WebDriverWait(v.browser, 7)
#Make it only 1 Window tab
for i in range(len(v.browser.window_handles) - 1):
v.browser.close()
v.browser.switch_to.window(v.browser.window_handles[0])
def __init__(self, *args, **kwargs):
chromedriver_autoinstaller.install()
self.sender = gmailSender(*args, **kwargs)
if kwargs['first_time']:
self.firstRun()
else:
self.run()
def returnChromeSettings():
browser=''
chromedriver_autoinstaller.install()
profile = {"plugins.plugins_list": [{"enabled": True, "name": "Chrome PDF Viewer"}], # Disable Chrome's PDF Viewer
"download.default_directory": completeDownloadFolder ,
"download.prompt_for_download": False,
"download.directory_upgrade": True,
"download.extensions_to_open": "applications/pdf",
"plugins.always_open_pdf_externally": True #It will not show PDF directly in chrome
}
if objControl.heroku:
#Chrome configuration for heroku
chrome_options= webdriver.ChromeOptions()
chrome_options.binary_location=os.environ.get("GOOGLE_CHROME_BIN")
chrome_options.add_argument("--headless")
chrome_options.add_argument("--disable-dev-shm-usage")
chrome_options.add_argument("--no-sandbox")
chrome_options.add_experimental_option('prefs',profile)
browser=webdriver.Chrome(executable_path=os.environ.get("CHROMEDRIVER_PATH"),chrome_options=chrome_options)
else:
options = Options()
options.add_experimental_option("prefs", profile)
browser=webdriver.Chrome(options=options)
return browser
def run(browser: str="", mute_browser: bool=True, headless: bool=True) -> webdriver:
# Search for browsers
system = platform.system()
browsers = {
"firefox": 0,
"chrome": 0,
"edge": 0,
}
webdrivers = {
"geckodriver": 0,
"chromedriver": 0,
}
if browser == "firefox":
browsers['firefox'] = 1
elif browser == "chrome":
browsers["chrome"] = 1
else:
if system == "Linux":
from distutils.spawn import find_executable
if find_executable("firefox"):
if find_executable('geckodriver'):
webdrivers['geckodriver'] = 1
browsers["firefox"] = 1
if find_executable("chromium"):
if find_executable('chromedriver'):
webdrivers['chromedriver'] = 1
browsers["chrome"] = 1
elif system == "Windows":
import os
if os.path.isdir("C:\\Program Files\\Google\\Chrome"):
browsers["chrome"] = 1
if os.path.isdir("C:\\Program Files (x86)\\Google\\Chrome"):
browsers["chrome"] = 1
if os.path.isdir("C:\\Program Files\\Mozilla Firefox"):
browsers["firefox"] = 1
if os.path.isdir("C:\\Program Files (x86)\\Mozilla Firefox"):
browsers["firefox"] = 1
if browsers["chrome"] == 1:
if not webdrivers['chromedriver'] == 1:
import chromedriver_autoinstaller
chromedriver_autoinstaller.install()
options = webdriver.ChromeOptions()
if headless:
options.headless = True
if mute_browser:
options.add_argument("--mute-audio")
return webdriver.Chrome(chrome_options=options)
elif browsers["firefox"] == 1:
if not webdrivers['geckodriver'] == 1:
import geckodriver_autoinstaller
geckodriver_autoinstaller.install()
options = webdriver.FirefoxProfile()
if headless:
options.headless = True
if mute_browser:
options.set_preference("media.volume_scale", "0.0")
return webdriver.Firefox(firefox_profile=options)
def main():
display_warning()
# Option to block the notification popup (for Google Chrome)
option = Options()
option.add_argument("--disable-infobars")
option.add_argument("start-maximized")
option.add_argument("--disable-extensions")
# Pass the argument 1 to allow and 2 to block
option.add_experimental_option(
"prefs", {"profile.default_content_setting_values.notifications": 2})
# Iterate the loop until the user enters correct email and password
while True:
try:
# Login info (Facebook)
email_adrs = input('Email: ')
password = getpass()
# Automatically installs the chrome driver
chromedriver_autoinstaller.install()
driver = webdriver.Chrome()
driver.get(
"https://www.facebook.com/v2.6/dialog/oauth?redirect_uri=fb464891386855067%3A%2F%2Fauthorize%2F&scope=user_birthday%2Cuser_photos%2Cuser_education_history%2Cemail%2Cuser_relationship_details%2Cuser_friends%2Cuser_work_history%2Cuser_likes&response_type=token%2Csigned_request&client_id=464891386855067&ret=login&fallback_redirect_uri=221e1158-f2e9-1452-1a05-8983f99f7d6e&ext=1556057433&hash=Aea6jWwMP_tDMQ9y"
)
# Email input
email_text = driver.find_element_by_id("email")
email_text.send_keys(email_adrs)
# Password input
password_text = driver.find_element_by_id("pass")
password_text.send_keys(password)
# Submit
password_text.submit()
# Get the continue button
time.sleep(3)
continue_button = driver.find_element_by_css_selector(
'.oajrlxb2.g5ia77u1.qu0x051f.esr5mh6w.e9989ue4.r7d6kgcz.rq0escxv.nhd2j8a9.pq6dq46d.p7hjln8o.kvgmc6g5.cxmmr5t8.oygrvhab.hcukyx3x.jb3vyjys.rz4wbd8a.qt6c0cv9.a8nywdso.i1ao9s8h.esuyzwwr.f1sip0of.lzcic4wl.n00je7tq.arfg74bv.qs9ysxi8.k77z8yql.l9j0dhe7.abiwlrkh.p8dawk7l.cbu4d94t.taijpn5t.k4urcfbm'
)
break
except Exception as e:
driver.close()
print('Incorrect credential, try again...\n')
continue_button.click()
# page_source property has the html source
long_token = parse_auth(driver.page_source)
# Close the driver for the long token
driver.close()
# Print out the XAuthToken
xauth = get_xauth_token(long_token)['data']['api_token']
print(f"\nXAuthToken: {xauth}")
def run_stuff():
chromedriver_autoinstaller.install()
chrome_options = Options()
chrome_options.add_argument("--headless")
chrome_options.add_argument('log-level=3')
inputs = 0
driver = webdriver.Chrome(options=chrome_options)
driver.get("https://newegg.com/product-shuffle")
time.sleep(1)
driver.find_element_by_xpath("//strong[text()='Sign Up']").click()
inputElement = driver.find_elements_by_xpath("//input")
for inputEl in inputElement:
inputs = inputs + 1
if (inputs == 1):
inputEl.send_keys(random_generator(5))
elif (inputs == 2):
inputEl.send_keys(random_generator(5))
elif (inputs == 3):
inputEl.send_keys('newegg' + random_generator(10) + email)
elif (inputs == 7):
inputEl.send_keys(password)
driver.find_element_by_xpath("//button[@class='btn btn-orange']").click()
time.sleep(2)
cards = driver.find_elements_by_xpath(
"//div[@class='multiple-choice-card ']")
for card in cards:
try:
card.click()
except Exception as e:
print(e)
continue
try:
continue1 = driver.find_element_by_xpath(
"//button[contains(@class,'btn btn-primary')]")
continue1.click()
except:
print("Exception")
time.sleep(0.5)
try:
continue2 = driver.find_element_by_xpath(
"//button[@class='btn btn-primary']")
continue2.click()
except:
print("Exception")
time.sleep(1)
print("Entered Shuffle")
driver.close()
def chromedriver():
chrome_version = chromedriver_autoinstaller.get_chrome_version()
logger.debug("Google Chrome version {0} is installed.".format(chrome_version))
try:
logger.debug("Checking and/or installing chromedriver")
chromedriver_autoinstaller.install()
finally:
logger.debug("Chromedriver is ok.")
def get_browser():
"""Load the browser"""
global browser
chromedriver_autoinstaller.install()
# Register the driver
browser = webdriver.Chrome()
# browser.get("https://www.linkedin.com/uas/login?session_redirect=https%3A%2F%2Fwww%2Elinkedin%2Ecom%2Ffeed%2F&fromSignIn=true&trk=cold_join_sign_in")
browser.get(r"https://linkedin.com")
def drUpdater():
try:
print("'chromedriver' indiriliyor...")
chromedriver_autoinstaller.install("./")
except:
input(
"\nInternet bağlantınızı kontrol edin !\nKapatmak için 'enter'a basınız..."
)
exit()
def get(self, request):
# Comprobar speech activo.
try:
chromedriver_autoinstaller.install()
except Exception as e:
print('ERROR', e)
return render(request, 'index.html')
def env_setup(data_setup):
global driver
chromedriver_autoinstaller.install()
driver = WebDriver()
url = data.get("url")
driver.maximize_window()
driver.get(url)
yield
driver.close()
def __init__(self, title, graph, coords, export_path):
self.title = title
self.width, self.height = 800, 532
self.graph = graph
self.coords = coords
self.export_path = export_path
self.plot = self.create_plot()
chromedriver_autoinstaller.install(cwd=True)
geckodriver_autoinstaller.install(cwd=True)
def make_webdriver():
# Import selenium here because it's slow to import
from selenium import webdriver
import chromedriver_autoinstaller
chromedriver_autoinstaller.install()
driver = webdriver.Chrome()
atexit.register(lambda: driver.quit())
return driver
def main():
# Check if the current version of chromedriver exists
# and if it doesn't exist, download it automatically,
# then add chromedriver to path
try:
chromedriver_autoinstaller.install()
except FileNotFoundError:
print(
"ChromeDriver Autoinstaller failed, please install the correct 'chromedriver' manually."
)
nltk.download('stopwords')
again = True
while again:
if easygui.ynbox("Would you like to clean program data?", 'Title',
('Yes', 'No')):
shutil.rmtree("./report", ignore_errors=True)
shutil.rmtree("./most_used", ignore_errors=True)
shutil.rmtree("./bar_charts", ignore_errors=True)
if os.path.isfile("scraped.json"):
os.remove("scraped.json")
if os.path.isfile("found_profile_links.json"):
os.remove("found_profile_links.json")
create_folder("./report/")
create_folder("./most_used/")
create_folder("./bar_charts/")
msg = "Enter your subject profile name"
title = "Social Media Scraper"
fieldNames = ["Subject Profile Name"]
fieldValues = easygui.multenterbox(msg, title, fieldNames)
if fieldValues is None:
exit(0)
# make sure that none of the fields were left blank
while 1:
errmsg = ""
for i, name in enumerate(fieldNames):
if fieldValues[i].strip() == "":
errmsg += "{} is a required field.\n\n".format(name)
if errmsg == "":
break # no problems found
fieldValues = easygui.multenterbox(errmsg, title, fieldNames,
fieldValues)
if fieldValues is None:
break
print("Profile name is: {}".format(fieldValues))
profile_name = fieldValues[0]
find_profiles(profile_name)
scrape()
natural_language()
again = easygui.ynbox("Would you like to try again?", 'Title',
('Yes', 'No'))
def startSession():
print("PROGRAMA INICIADO")
"""pathProject = getProjectFolder()"""
# browser = "chromedriver.exe"
# fullRoute = getProjectFolder() + "\\" + browser
chromedriver_autoinstaller.install()
webBrowser = webdriver.Chrome()
global browserDriver
browserDriver = webBrowser
def create_driver(phone=False):
chromedriver_autoinstaller.install()
options = webdriver.ChromeOptions()
options.add_argument('user-data-dir=' + os.getcwd() + '\\chrome_profile')
# Emulate phone driver for instagram
if phone:
options.add_experimental_option("mobileEmulation",
{"deviceName": "Pixel 2"})
return webdriver.Chrome(options=options)
def __init__(self):
chromedriver_autoinstaller.install()
options = Options()
options.add_argument('--headless')
options.add_argument('--disable-gpu')
options.add_argument("window-size=1920,1080")
self.driver = webdriver.Chrome()
self.driver.get("https://www.epicgames.com/store/fr/")
def initialize():
chrome_ver = chromedriver_autoinstaller.get_chrome_version().split('.')[
0] # 크롬드라이버 버전 확인
try:
driver = webdriver.Chrome(f'./{chrome_ver}/chromedriver.exe')
except:
chromedriver_autoinstaller.install(True)
driver = webdriver.Chrome(f'./{chrome_ver}/chromedriver.exe')
driver.implicitly_wait(10)
def __init__(self):
chrome_options = webdriver.ChromeOptions()
chromedriver_autoinstaller.install()
profile = os.path.join(os.getcwd(), "profile", "wpp")
#chrome_options.add_argument("--user-data-dir=./tmp/browser/")
chrome_options.add_argument(r"user-data-dir={}".format(profile))
chrome_options.add_argument("--profile-directory=Default")
self.driver = webdriver.Chrome(chrome_options=chrome_options)
def navegador():
"""Crea una ruta ejecutable para chromedriver"""
try:
return webdriver.Chrome()
except SessionNotCreatedException:
print("Actualizando Chromedriver")
chromedriver_autoinstaller.install()
return webdriver.Chrome()
except WebDriverException:
print("Comprobando la instalación de Chromedriver")
chromedriver_autoinstaller.install()
return webdriver.Chrome()
def initWebDriver():
'''check Chrome driver version '''
chromedriver_autoinstaller.install(cwd=True)
#set downLoad path
path = os.path.dirname(os.path.abspath(__file__)) + '\\resource'
options = webdriver.ChromeOptions()
options.add_experimental_option("prefs",
{"download.default_directory": path})
# driver Global
global driver
driver = webdriver.Chrome(options=options) #return driver
def get_driver():
chromedriver_autoinstaller.install()
options = webdriver.ChromeOptions()
options.add_argument("--disable-blink-features=AutomationControlled")
options.add_experimental_option("excludeSwitches", ["enable-automation"])
options.add_experimental_option('useAutomationExtension', False)
options.add_argument("--disable-extensions")
options.add_argument("--enable-javascript")
options.add_argument("--disable-popup-blocking")
return webdriver.Chrome(options=options)
