def get_basic_setup() -> BrowserSetup:
browser_setup = BrowserSetup()
if properties.OS_LINUX in platform:
print(properties.OS_LINUX)
options = Options()
options.accept_untrusted_certs = True
options.assume_untrusted_cert_issuer = True
browser_setup.options = options
# browser_setup.web_home_path = "/static/index.html"
browser_setup.driver_path = properties.DRIVER_LINUX_PATH
elif properties.OS_MAC in platform:
print(properties.OS_MAC)
options = Options()
options.accept_untrusted_certs = True
options.assume_untrusted_cert_issuer = True
browser_setup.options = options
# browser_setup.web_home_path = "/static/index.html"
browser_setup.driver_path = properties.DRIVER_MAC_PATH
elif properties.OS_WINDOW in platform:
print(properties.OS_WINDOW)
options = Options()
options.accept_untrusted_certs = True
options.assume_untrusted_cert_issuer = True
options.add_argument(properties.USER_DATA_DIR)
browser_setup.options = options
# browser_setup.web_home_path = "\\static\\index.html"
browser_setup.driver_path = properties.DRIVER_WINDOWS_PATH
else:
raise UnsupportedPlatform
return browser_setup
def login(self, ip=None, password=None):
self.baseurl = "https://" + ip
options = Options()
options.set_capability("acceptInsecureCerts", True)
options.accept_untrusted_certs = True
options.add_argument('--ignore-certificate-errors')
if "systest-dev1" == gethostname():
options.headless = True
now = datetime.now()
now = now.strftime("%d-%b-%Y-%H:%M:%S.%f")
self.wd = webdriver.Chrome(options=options,
service_args=[
"--verbose", "--log-path=/tmp/" + now +
" chromedriver.log"
])
self.wd.set_window_size(1366, 758)
self.wd.implicitly_wait(10)
time.sleep(1)
url = self.baseurl + "#/admin"
self.wd.get(url)
element = self.wd.find_element_by_id("admin-tab")
element.click()
time.sleep(1)
element = self.wd.find_element_by_id("passwordInput")
element.clear()
element.send_keys(password)
time.sleep(1)
element = self.wd.find_element_by_id("passEnter")
self.wd.execute_script("document.getElementById('passEnter').click()")
# not so fast...
time.sleep(2)
def stat_scraper(link_list, driver):
#truncate_table(table_name)
options = Options()
options.accept_untrusted_certs = True
options.assume_untrusted_cert_issuer = True
options.add_argument('--load-extension=/Users/Philip/Documents/NBA prediction script/Incremental Pipelines/3.34.0_0')
browser = webdriver.Chrome(executable_path=driver, chrome_options=options)
browser.get(link_list[0])
#browser.find_element_by_xpath('/html/body/main/div[2]/div/div[2]/div/div/div[1]/div[1]/div/div/label/select/option[1]').click()
try:
wait = WebDriverWait(browser, 120).until(EC.presence_of_element_located((By.XPATH, '/html/body/main/div[2]/div/div[2]/div/div/nba-stat-table/div[1]/div/div/select')))
except TimeoutException or NoSuchElementException:
browser.refresh()
#logging.info('Failed to connect to page')
pages = browser.find_element_by_xpath('/html/body/main/div[2]/div/div[2]/div/div/nba-stat-table/div[1]/div/div/select')
options = [x for x in pages.find_elements_by_tag_name("option")]
for page in range(2, len(options)+1):
browser.find_element_by_xpath('/html/body/main/div[2]/div/div[2]/div/div/nba-stat-table/div[3]/div/div/select/option[{}]'.format(str(page))).click()
table = browser.find_element_by_class_name('nba-stat-table')
content = table.get_attribute('innerHTML')
df = pd.read_html(content)[0]
f_df = format_matchup(df)
f_df['Game Date'] = f_df.loc[:, 'Game Date'].apply(convert_date)
insert_into_database(f_df, link_list[1])
browser.quit()
return
def gen_browser(link, driver):
options = Options()
options.accept_untrusted_certs = True
options.assume_untrusted_cert_issuer = True
options.add_argument('--load-extension=/Users/Philip/Documents/NBA prediction script/Incremental Pipelines/3.34.0_0')
browser = webdriver.Chrome(executable_path=driver, chrome_options=options)
browser.get(link)
try:
wait = WebDriverWait(browser, 120).until(EC.presence_of_element_located((By.XPATH, '/html/body/main/div[2]/div/div[2]/div/div/nba-stat-table/div[1]/div/div/select')))
except TimeoutException or NoSuchElementException:
browser.refresh()
logging.info('[FAILED TO CONNECT TO PAGE]: {}'.format(link))
return browser
def do_screenshots_of_target_hosts():
show_time("starting screenshoting targets")
options = Options()
options.headless = True
options.accept_untrusted_certs = True
options.assume_untrusted_cert_issuer = True
options.add_argument("--no-sandbox")
options.add_argument('--disable-dev-shm-usage')
driver = webdriver.Chrome(options=options)
os.makedirs(args.domain, exist_ok=True)
for key, value in ip_addresses.items():
if 80 in value:
if value[80]['has_content']:
execute_browser('http://', key, value[80]['start_url'], driver,
value['source'], value[80]['http_status_code'])
if 443 in value:
if value[443]['has_content']:
execute_browser('https://', key, value[443]['start_url'],
driver, value['source'],
value[443]['http_status_code'])
driver.close()
driver.quit()
show_time("screenshoting targets finished")
def run(self):
try:
import os
driverName = "\\chromedriver.exe"
# defining base file directory of chrome drivers
driver_loc = os.path.dirname(
os.path.abspath(__file__)) + "\\ChromeDriver\\"
# defining the file path of your exe file automatically updating based on your browsers current version of chrome.
currentPath = driver_loc + chrome_browser_version + driverName
# check if new version of drive exists --> only continue if it doesn't
Newpath = driver_loc + nextVersion
# check if we have already downloaded the newest version of the browser
newfileloc = Newpath + driverName
newpathexists = os.path.exists(newfileloc)
if newpathexists == False:
try:
# open chrome driver and attempt to download new chrome driver exe file.
# set the arguments and options
chromeOptions = Options()
chromeOptions.add_experimental_option(
"prefs",
{
"download.default_directory": driver_loc,
"download.prompt_for_download": False,
"download.directory_upgrade": True,
"safebrowsing.enabled": True,
"profile.managed_default_content_settings.images":
2,
},
)
chromeOptions.add_experimental_option(
"excludeSwitches", ["enable-logging"])
chromeOptions.add_argument("--headless")
chromeOptions.add_argument(
"--blink-settings=imagesEnabled=false")
chromeOptions.add_argument("--disable-popup-blocking")
chromeOptions.add_argument("--ignore-certificate-errors")
chromeOptions.add_argument("--allow-insecure-localhost")
chromeOptions.add_argument(
"--allow-running-insecure-content")
chromeOptions.accept_untrusted_certs = True
chromeOptions.assume_untrusted_cert_issuer = True
service_args = ["hide_console"]
try:
print("~~~Calling Update Driver")
update_driver = Chrome(
executable_path=currentPath,
options=chromeOptions,
service_args=service_args,
)
print("~~~Update Driver Opened")
# opening up url of chromedriver to get new version of chromedriver.
chromeDriverURL = (
"https://chromedriver.storage.googleapis.com/index.html?path="
+ nextVersion)
update_driver.set_page_load_timeout(10)
update_driver.delete_all_cookies()
update_driver.get(chromeDriverURL)
print("~~~Update Website Got")
# time.sleep(5)
event.wait(5)
# find records of table rows
table = update_driver.find_elements_by_css_selector(
"tr")
# check the length of the table
Table_len = len(table)
# ensure that table length is greater than 4, else fail. -- table length of 4 is default when there are no availble updates
if Table_len > 4:
# define string value of link
rowText = table[(len(table) - 2)].text[:6]
# time.sleep(1)
event.wait(1)
# select the value of the row
update_driver.find_element_by_xpath(
"//*[contains(text()," + '"' + str(rowText) +
'"' + ")]").click()
event.wait(1)
# time.sleep(1)
# select chromedriver zip for windows
update_driver.find_element_by_xpath(
"//*[contains(text()," + '"' + "win32" + '"' +
")]").click()
print("~~~Download Started")
# time.sleep(5)
event.wait(5)
update_driver.quit()
print("~~~Update Driver Exited")
try:
from zipfile import ZipFile
import shutil
fileName = os.path.join(
os.path.dirname(driver_loc),
"chromedriver_win32.zip",
)
# Create a ZipFile Object and load sample.zip in it
with ZipFile(fileName, "r") as zipObj:
# Extract all the contents of zip file in different directory
zipObj.extractall(Newpath)
print("~~~Newer Version Extracted")
except Exception as ex:
print(
"Error in extracting:\t\tAn exception of type {0} occurred. Arguments:\n{1}"
.format(type(ex).__name__, ex.args))
try:
# delete downloaded file
os.remove(fileName)
print("Downloaded Zip Deleted")
except Exception as ex:
print(
"~~~Error in deleting zip:\t\tAn exception of type {0} occurred. Arguments:\n{1}"
.format(type(ex).__name__, ex.args))
# defining old chrome driver location
oldPath = driver_loc + lastVersion
oldpathexists = os.path.exists(oldPath)
# this deletes the old folder with the older version of chromedriver in it
if oldpathexists == True:
try:
import stat
shutil.rmtree(oldPath, ignore_errors=True)
print("Old Version Deleted")
except Exception as ex:
print(
"~~~Error in deleting previous version:\t\tAn exception of type {0} occurred. Arguments:\n{1}"
.format(type(ex).__name__, ex.args))
else:
# update_driver.quit()
print("~~~No new version available")
except Exception as ex:
print(
"~~~Error in update driver:\t\tAn exception of type {0} occurred. Arguments:\n{1}"
.format(type(ex).__name__, ex.args))
finally:
try:
# close the driver
update_driver.quit()
print("~~~Update Driver Exited")
except Exception as ex:
print(
"~~~Error in quitting:\t\tAn exception of type {0} occurred. Arguments:\n{1}"
.format(type(ex).__name__, ex.args))
except Exception as ex:
print(
"~~~Error in if:\t\tAn exception of type {0} occurred. Arguments:\n{1}"
.format(type(ex).__name__, ex.args))
else:
print("~~~ChromeDriver Upto Date~~~")
except Exception as ex:
print(
"~~~Error in update:\t\tAn exception of type {0} occurred. Arguments:\n{1}"
.format(type(ex).__name__, ex.args))
def run(self):
"""
Will check wheather the portal in logged in after a set interval and will sign in if not.
"""
try:
# set the arguments and options
chromeOptions = Options()
prefs = {"profile.managed_default_content_settings.images": 2}
chromeOptions.add_experimental_option("prefs", prefs)
chromeOptions.add_experimental_option("excludeSwitches",
["enable-logging"])
chromeOptions.add_argument("--headless")
chromeOptions.add_argument("--blink-settings=imagesEnabled=false")
chromeOptions.add_argument("--disable-popup-blocking")
chromeOptions.add_argument("--ignore-certificate-errors")
chromeOptions.add_argument("--allow-insecure-localhost")
chromeOptions.add_argument("--allow-running-insecure-content")
chromeOptions.accept_untrusted_certs = True
chromeOptions.assume_untrusted_cert_issuer = True
service_args = ["hide_console"]
currentPath = (os.path.dirname(os.path.abspath(__file__)) +
"\\ChromeDriver\\" + chrome_browser_version +
"\\chromedriver.exe")
while True:
try:
print("\nCalling Driver")
# Creating an instance of chrome
driver = Chrome(
executable_path=currentPath,
options=chromeOptions,
service_args=service_args,
)
print("Driver Called")
driver.set_page_load_timeout(10)
driver.delete_all_cookies()
# open a page
driver.get("Enter Checking Site Here")
print("Getting Site")
try:
"""
remove this try except if the your wifi doesn't block websites
"""
# xpath if the website is blocked
element = driver.find_element_by_xpath(
"Enter xpath to an element in the blocked page")
print("Site Blocked\n")
except:
try:
# xpath to any thing in the website to make sure you are connected to the internet
element = driver.find_element_by_xpath(
"/Enter xpath to an element in the page")
print("Site Opening\n")
except:
try:
"""
if your portal doesn't have auto redirect, uncomment the following line and type in your login url
"""
# driver.get("Paste Login Webiste URL Here")
# change the ids to those in your login website
# you can use developer mode to find the id of fields (use ctrl + shift + i)
# change the username and password to the required one
print("Trying To Login")
# select usnername field
element = driver.find_element_by_id(
"Ending id of user input field")
print("User Found")
element.send_keys("Enter username")
print("User Inputted")
# select password field
element = driver.find_element_by_id(
"Ending id of password input field")
print("Passwprd Found")
element.send_keys("Enter password")
print("Password Inputted")
# select submit button
element = driver.find_element_by_id(
"Enter id of submit button")
print("Button Found")
element.click()
print("Logged In\n")
# except NoSuchElementException as ex:
# print("Can't Login")
# event.wait(120)
except Exception as ex:
print(
"Can't login:\t\tAn exception of type {0} occurred. Arguments:\n{1}"
.format(type(ex).__name__, ex.args))
event.wait(60)
continue
except Exception as ex:
print(
"Error in loop:\t\tAn exception of type {0} occurred. Arguments:\n{1}"
.format(type(ex).__name__, ex.args))
try:
driver.quit()
except Exception as ex:
print(
"Error in Quitting:\t\tAn exception of type {0} occurred. Arguments:\n{1}"
.format(type(ex).__name__, ex.args))
event.wait(60)
continue
try:
driver.quit()
except Exception as ex:
print(
"Error in Quitting in loop:\t\tAn exception of type {0} occurred. Arguments:\n{1}"
.format(type(ex).__name__, ex.args))
event.wait(300)
continue
except Exception as ex:
print(
"Error outside loop:\t\tAn exception of type {0} occurred. Arguments:\n{1}"
.format(type(ex).__name__, ex.args))
finally:
try:
driver.quit()
except Exception as ex:
print(
"Error in Quitting in final:\t\tAn exception of type {0} occurred. Arguments:\n{1}"
.format(type(ex).__name__, ex.args))
finally:
print("Login Thread Exited")
def login(email):
global valid
global invalid
global count
colors = {
'merah': Fore.RED,
'hijau': Fore.GREEN,
'putih': Fore.WHITE,
'kuning': Fore.YELLOW,
'biru': Fore.CYAN,
'black': Fore.BLACK
}
if os.system == "nt":
path = "chromedriver.exe"
else:
path = "chromedriver"
url = "https://www.remitly.com/us/en/users/register"
f = open("proxies.txt", "r")
proxies = f
for proxi in proxies:
myproxy = proxi
prox = Proxy()
prox.proxy_type = ProxyType.MANUAL
prox.http_proxy = myproxy
prox.socks5_proxy = myproxy
prox.ssl_proxy = myproxy
capabilities = webdriver.DesiredCapabilities.CHROME
prox.add_to_capabilities(capabilities)
options = Options()
options.headless = True
options.add_argument('log-level=3')
options.accept_untrusted_certs = True
options.assume_untrusted_cert_issuer = True
options.add_experimental_option('excludeSwitches', ['enable-logging'])
options.add_argument("--no-sandbox")
options.add_argument("--disable-impl-side-painting")
options.add_argument('--disable-gpu')
options.add_argument("--disable-setuid-sandbox")
options.add_argument("--disable-seccomp-filter-sandbox")
options.add_argument("--disable-breakpad")
options.add_argument("--disable-client-side-phishing-detection")
options.add_argument("--disable-cast")
options.add_argument("--disable-cast-streaming-hw-encoding")
options.add_argument("--disable-cloud-import")
options.add_argument("--disable-popup-blocking")
options.add_argument("--ignore-certificate-errors")
options.add_argument("--disable-session-crashed-bubble")
options.add_argument("--disable-ipv6")
options.add_argument("--allow-http-screen-capture")
options.add_argument("--start-maximized")
driver = webdriver.Chrome(path,
chrome_options=options,
desired_capabilities=capabilities)
driver.delete_all_cookies()
driver.get(url)
driver.find_element_by_name('email_address').send_keys(email)
driver.find_element_by_name('password').send_keys('jakarta321')
Select(
driver.find_element_by_xpath(
'/html/body/div[2]/div[2]/div/div/form/div[3]/div/select')
).select_by_value('4')
button = driver.find_element_by_xpath(
'/html/body/div[2]/div[2]/div/div/form/div[4]/button')
button.click()
# response = driver.find_elements_by_xpath("/html/body/div[2]/div[2]/div/div/script[1]")
sleep(1)
response = driver.execute_script(
"return document.getElementsByTagName('html')[0].innerHTML")
if 'This email address is associated with another account.' in (
str(response)):
with myLock:
Pac = email
loglive.write(Pac)
count = count + 1
valid = valid + 1
print("LIVE => " + email)
else:
with myLock:
logdie.write(email)
count = count + 1
invalid = invalid + 1
print("DIE => " + email)
def initialize_driver(headless=True,
with_ublock=False,
chromedriver_path=None):
if not chromedriver_path:
try:
chromedriver_path = chromedriver_autoinstaller.install()
except Exception as exception:
log.critical(
f"Failed to install the built-in chromedriver: {exception}\n"
"download the correct version for your system at "
"https://chromedriver.chromium.org/downloads and use the"
"--chromedriver argument to point to the chromedriver executable"
)
sys.exit()
log.info(f"Initialising chromedriver at {chromedriver_path}...")
chrome_options = Options()
chrome_options.add_argument(
"--disable-blink-features=AutomationControlled")
chrome_options.add_experimental_option(
"excludeSwitches", ["enable-automation", "enable-logging"])
chrome_options.add_experimental_option('useAutomationExtension', False)
chrome_options.add_argument('--disable-dev-shm-usage')
chrome_options.accept_untrusted_certs = True
chrome_options.assume_untrusted_cert_issuer = True
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument("--allow-http-screen-capture")
chrome_options.add_argument("--disable-impl-side-painting")
chrome_options.add_argument("--disable-setuid-sandbox")
chrome_options.add_argument("--disable-seccomp-filter-sandbox")
if headless:
chrome_options.add_argument("--headless")
chrome_options.add_argument("window-size=1920,1080")
chrome_options.add_argument("--log-level=3")
chrome_options.add_argument("--silent")
chrome_options.add_argument("--disable-logging")
# allows selenium to accept cookies with a non-int64 'expiry' value
chrome_options.add_experimental_option("w3c", False)
# removes the 'DevTools listening' log message
chrome_options.add_experimental_option("excludeSwitches",
["enable-logging"])
if with_ublock:
chrome_options.add_extension(
os.path.join(os.getcwd(), "bin", "ublock", "ublock-extension.crx"))
logs_path = os.path.join(os.getcwd(), "logs")
if not (os.path.isdir(logs_path)):
os.makedirs(logs_path)
driver = webdriver.Chrome(
executable_path=chromedriver_path,
service_log_path=os.path.join(logs_path, "webdrive.log"),
# Don't verify self-signed cert, should help with 502 errors
# (https://github.com/wkeeling/selenium-wire/issues/55)
# seleniumwire_options={"verify_ssl": False},
options=chrome_options,
)
if with_ublock:
log.debug("Configuring uBlock")
# set up uBlock
driver.get(
"chrome-extension://ilchdfhfciidacichehpmmjclkbfaecg/settings.html"
)
# Un-hide the file upload button so we can use it
element = driver.find_elements_by_class_name("hidden")
driver.execute_script(
"document.getElementsByClassName('hidden')[0].className = ''",
element)
# scroll down (for debugging)
driver.execute_script("window.scrollTo(0, 2000)")
uBlock_settings_file = str(
os.path.join(os.getcwd(), "bin", "ublock", "ublock-settings.txt"))
driver.find_element_by_id("restoreFilePicker").send_keys(
uBlock_settings_file)
try:
WebDriverWait(driver, 3).until(EC.alert_is_present())
# click ok on pop up to accept overwrite
driver.switch_to.alert.accept()
except TimeoutException as ex:
log.error("Timeout waiting for ublock config overwrite alert")
# leave uBlock config
driver.get("about:blank")
return driver