retweets_num = card.find_element_by_xpath(
'.//div[@data-testid="retweet"]').text
curtidas_num = card.find_element_by_xpath(
'.//div[@data-testid="like"]').text
tweet = (apelido, username, data_postagem, respondendo, comentario,
link_img, comentarios_num, retweets_num, curtidas_num)
return tweet
options = EdgeOptions()
options.use_chromium = True
options.headless = True
driver = Edge(options=options)
driver.get('https://twitter.com/search?q=vacina&src=typed_query')
driver.find_element_by_link_text('Mais recentes').click()
data = []
tweet_ids = set()
last_position = driver.execute_script("return window.pageYOffset;")
scrolling = True
scroll_attempt = 0
i = 0
while scrolling:
page_cards = driver.find_elements_by_xpath('//div[@data-testid="tweet"]')
for card in page_cards[-15:]:
tweet = get_tweet_data(card)
if tweet:
tweet_id = ''.join(tweet)
NetIM - Synthetic Test
Script: Edge-open-url-sample.py
Application: Edge
Simple sample showing how to automate the Edge browser on a windows machine to navigate to a page
Usage:
python Edge-open-url-sample.py
"""
import time
# Configure Selenium
from msedge.selenium_tools import Edge, EdgeOptions, EdgeService
MSEDGEDRIVER_PATH = "C:\\edgedriver_win64\\msedgedriver.exe"
if __name__ == "__main__":
service = EdgeService(MSEDGEDRIVER_PATH)
options = EdgeOptions()
options.use_chromium = True
driver = Edge(executable_path=MSEDGEDRIVER_PATH, options=options)
# Synthetic test
url = "http://www.riverbed.com"
driver.get(url)
time.sleep(5)
driver.close()
driver.quit()
def chooseAccount():
with open('data.txt') as json_file:
data = json.load(json_file)
userInfo ='account: ' + data['username']
print(userInfo)
userName = data['username']
passWord = data['password']
print("link:")
link = input()
print("number of photos: ")
amount = input()
# format text and amount
amount = int(amount)
# auto login
options = EdgeOptions()
options.use_chromium = True
options.add_argument('headless')
driver = Edge('msedgedriver', options = options)
driver.get(link)
time.sleep(2)
userForm = driver.find_element_by_css_selector("input[name='username']")
passForm = driver.find_element_by_css_selector("input[name='password']")
userForm.send_keys(userName)
passForm.send_keys(passWord)
driver.find_element_by_css_selector("button[type='submit']").click()
time.sleep(3)
driver.execute_script("document.querySelector('.sqdOP.yWX7d.y3zKF').click()")
# get link image to list
time.sleep(2)
if amount > 1:
spriteBtn = driver.find_element_by_css_selector(".coreSpriteRightChevron")
list_link = []
def get_url1():
list_element = driver.find_elements_by_css_selector("img[style='object-fit: cover;']")
for image in list_element[:1]:
src = image.get_attribute("src")
list_link.append(src)
def get_url2():
list_element = driver.find_elements_by_css_selector("img[style='object-fit: cover;']")
list_element.pop(0)
for image in list_element[:1]:
src = image.get_attribute("src")
list_link.append(src)
for x in range(0, amount+1):
if (len(list_link) > 0):
get_url2()
else:
get_url1()
if len(list_link) == amount:
break
elif spriteBtn:
spriteBtn.click()
else:
break
time.sleep(0.5)
# check old image folder exist
if (os.path.isdir("./image")):
rmtree("./image")
# create new image folder
folderPath = os.getcwd()
folderPath += '\image'
os.mkdir(folderPath)
# clear screen
clear = lambda: os.system('cls')
clear()
for i in tqdm(range(100)):
pass
print("\nnumber of photos:", len(list_link))
pos = 0
for href in list_link:
print(pos+1, "DONE")
imagePathResult = "./image/image_" + str(pos) + ".png"
try:
downloadFile(href)
copy("./image/image.png", imagePathResult)
except:
print("error at %s" %pos+1)
pos += 1
os.remove("./image/image.png")
resultPath = os.getcwd()
resultPath = resultPath + '\image'
os.startfile(resultPath)
driver.close()
chooseMenu()
if (os.path.isfile(path)):
key = 2
else:
key = 1
menu(key)
def getRightStufAnimeData(memberStatus, title, bookType, currPageNum):
#Starts wevdriver to scrape edge chromium
options = EdgeOptions()
options.use_chromium = True
options.add_argument("-inprivate")
options.add_argument("--headless")
driver = Edge(options=options)
#Create a empty list for all the data types we want to track
titleList, priceList, stockStatusList = [], [], []
#Get the URL for the page we are going to scrape for data
driver.get(getPageURL(bookType, currPageNum, title))
#Need to wait so the website can finish loading
time.sleep(5)
#Parse the HTML to start scraping for data
soup = BeautifulSoup(driver.page_source, "html.parser")
#Get the Title, Price, and Stock Status Data of each Manga Volume and whether or not next page button exists
titleList = soup.find_all("span", {"itemprop" : "name"})
priceList = soup.find_all("span", {"itemprop" : "price"})
stockStatusList = soup.find_all("div", {"class" : "product-line-stock-container"})
nextPageButton = soup.find("li", {"class" : "global-views-pagination-next"})
#Check to see if the title given by the user generates a valid URL for RightStufAnime
if not titleList:
print("Error!!! Invalid Title, Use English Title Variant w/ Appropriate Spacing & Capitalization")
return
else: #If the URL is a "valid" RightStufAnime website URL
websiteName = "RightStufAnime"
gotAnimeDiscount = 0.05 #5% Manga discount
#Format data into a single list
for fullTitle, price, stockStatus in zip(titleList, priceList, stockStatusList): #get only the title and volume number for the series we are looking for
if deParseString(title) in deParseString(fullTitle.text): #Fixes issue with capitilization
if memberStatus: #If user is a member add discount
priceVal = float(price.text[1:])
priceText = "$" + str(round((priceVal - (priceVal * gotAnimeDiscount)), 2)) #Add discount
else:
priceText = price.text
stockCheck = stockStatus.text
if stockCheck.find("Out of Stock") != -1:
stockCheck = "Out of Stock"
elif stockCheck.find("No Longer Available") != -1:
stockCheck = "Out of Print"
elif stockCheck.find("Pre-Order") != -1:
stockCheck = "Pre-Order"
else:
stockCheck = "Available"
dataFile.append([fullTitle.text, priceText, stockCheck])
#Check to see if there is another page
if nextPageButton != None:
currPageNum += 1
print(title)
getRightStufAnimeData(memberStatus, title, bookType, currPageNum)
#Initialize the a CSV to write into w/ appropiate headers
csvFile = websiteName + "Data.csv"
with open (csvFile, "w", newline = "", encoding = "utf-8") as file:
writeToFile = csv.writer(file)
writeToFile.writerow(["Title", "Price", "Stock Status"])
writeToFile.writerows(natsorted(dataFile)) #Sort data by title and write to the file
driver.quit()
return csvFile
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
from selenium.common.exceptions import TimeoutException
import os, datetime, csv
import pickle
driverOptions = EdgeOptions()
driverOptions.use_chromium = True
driverOptions.binary_location = r"C:\Program Files (x86)\Microsoft\Edge\Application\msedge.exe"
driverOptions.add_argument("--headless")
driverOptions.add_experimental_option("excludeSwitches", ["enable-logging"])
browser = Edge(options=driverOptions)
try:
try:
browser.get('https://www.netztest.at/')
cookies = pickle.load(
open(os.path.join(os.path.dirname(__file__), 'cookies.pkl'), "rb"))
for cookie in cookies:
if cookie['name'] == 'RMBTuuid':
cookie['expiry'] = int(
(datetime.datetime.utcnow().replace(
hour=0, minute=0, second=0, microsecond=0) -
datetime.datetime(1970, 1, 1)).total_seconds() + 1209600)
browser.add_cookie(cookie)
print("added cookie ", cookie)
except Exception:
print("Couldn't load cookies")
browser.get('https://www.netztest.at/de/Test')
from time import sleep
from msedge.selenium_tools import Edge, EdgeOptions
#edge无头浏览器 phantomJs可用，已停止更新
options = EdgeOptions()
options.use_chromium = True
options.add_argument("headless")
options.add_argument("disable-gpu")
#防止打印无用信息 enable-automation规避检测 #最新版浏览器已无用
options.add_experimental_option("excludeSwitches",
['enable-automation', 'enable-logging'])
#谷歌无头 #谷歌88.0版本可用
# from selenium.webdriver import Chrome
# from selenium.webdriver import ChromeOptions
# options = ChromeOptions()
# chrome_options.add_argument('--headless')
# chrome_options.add_argument('--disable-gpu')
# options.add_experimental_option("excludeSwitches", ["enable-automation",'enable-logging'])
# options.add_argument("--disable-blink-features=AutomationControlled")
# options.add_experimental_option('useAutomationExtension', False)
# wd = Chrome(options=options)
wd = Edge(options=options)
wd.get('https://www.baidu.com')
print(wd.page_source)
sleep(2)
wd.quit()
class QCourse:
def __init__(self):
# 初始化options
self.prefs = {"download.default_directory": os.getcwd()}
self.options = EdgeOptions()
self.options.use_chromium = True
self.options.add_argument("log-level=3")
self.options.add_experimental_option('excludeSwitches',
['enable-logging'])
self.options.add_experimental_option('prefs', self.prefs)
self.options.add_argument("--mute-audio")
self.login_url = 'https://ke.qq.com/'
self.driver = Edge(executable_path='msedgedriver.exe',
options=self.options)
def login(self):
self.driver.get('https://ke.qq.com/')
self.driver.find_element_by_id('js_login').click()
time.sleep(1)
WebDriverWait(self.driver, 300).until_not(
EC.presence_of_element_located((By.CLASS_NAME, 'ptlogin-mask')))
dictCookies = self.driver.get_cookies()
jsonCookies = json.dumps(dictCookies)
with open('cookies.json', 'w') as f:
f.write(jsonCookies)
print('登陆成功！')
def close(self):
self.driver.close()
def get_video(self, video_url=None, path=None):
if not video_url:
print('请输入视频url！')
# os.chdir(BASE_DIR)
if not os.path.exists('cookies.json'):
self.login()
with open('cookies.json', 'r') as f:
listCookies = json.loads(f.read())
self.driver.get(video_url)
for cookie in listCookies:
self.driver.add_cookie({
'domain': '.ke.qq.com',
'httpOnly': cookie['httpOnly'],
'name': cookie['name'],
'path': '/',
'secure': cookie['secure'],
'value': cookie['value']
})
self.driver.get(video_url)
# 等待视频开始播放
WebDriverWait(self.driver, 300).until(
EC.presence_of_element_located((By.CLASS_NAME, 'loki-time')))
WebDriverWait(
self.driver,
300).until_not(lambda driver: driver.find_element_by_class_name(
'loki-time').get_attribute("innerHTML") == '00:00 / 00:00')
networks = self.driver.execute_script(
'return window.performance.getEntries()')
ts_url = key_url = ''
for network in networks:
if '.ts?start' in network.get('name'):
ts_url = network.get('name')
elif 'get_dk' in network.get('name'):
key_url = network.get('name')
title = self.driver.title
# catalog = self.driver.execute_script('return document.getElementsByClassName("task-item task-info active")'
# '[0].parentNode.firstElementChild.innerText')
# os.chdir(os.path.join(os.getcwd(), catalog))
download_single(ts_url, key_url, title, path)
from msedge.selenium_tools import Edge, EdgeOptions
from selenium.webdriver.common.keys import Keys
import time
import random
options = EdgeOptions()
options.use_chromium = True
driver = Edge(options=options)
driver.get("https://www.reddit.com/")
driver.maximize_window()
time.sleep(2)
driver.execute_script("window.scrollTo(0, 300)")
time.sleep(4)
driver.execute_script("window.scrollTo(300, 600)")
time.sleep(4)
driver.execute_script("window.scrollTo(600, 900)")
time.sleep(4)
driver.execute_script("window.scrollTo(900, 1200)")
time.sleep(4)
driver.execute_script("window.scrollTo(1200, 1500)")
time.sleep(4)
driver.execute_script("window.scrollTo(1500, 1800)")
time.sleep(4)
driver.execute_script("window.scrollTo(1800, 2100)")
time.sleep(4)
driver.execute_script("window.scrollTo(2100, 2400)")
time.sleep(4)
driver.close()
from msedge.selenium_tools import options
from msedge.selenium_tools import Edge, EdgeOptions
from time import sleep
options = EdgeOptions()
options.use_chromium = True
driver = Edge(options=options)
usr = "******"
pwd = "D9806f2a2b"
driver.get("https://www.facebook.com/")
print("Opened Facebook")
sleep(0.5)
username_box = driver.find_element_by_id("email")
username_box.send_keys(usr)
print("Email ID entered")
sleep(0.1)
password_box = driver.find_element_by_id("pass")
password_box.send_keys(pwd)
print("Password entered")
sleep(0.5)
enter_box = driver.find_element_by_id("u_0_b").click()
print("Login pressed")
def main():
searchtext = input()
num_requested = int(input())
number_of_scrolls = num_requested / 400 + 1
# number_of_scrolls * 400 images will be opened in the browser
if not os.path.exists(download_path + searchtext.replace(" ", "_")):
os.makedirs(download_path + searchtext.replace(" ", "_"))
url = "https://www.google.co.in/search?q="+searchtext+"&source=lnms&tbm=isch"
chrome_driver_path = "msedgedriver.exe"
browser_path = "C:\\Program Files (x86)\\Microsoft\\Edge Beta\\Application\\msedge.exe"
option = EdgeOptions()
option.binary_location = browser_path
driver = Edge(executable_path = chrome_driver_path, options = option)
driver.get(url)
headers = {}
headers['User-Agent'] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36"
extensions = {"jpg", "jpeg", "png", "gif"}
img_count = 0
downloaded_img_count = 0
for _ in range(int(number_of_scrolls)):
for __ in range(15):
driver.execute_script("window.scrollBy(0, 1000000)")
time.sleep(0.2)
time.sleep(0.5)
try:
driver.find_element_by_xpath(
"//input[@value='Show more results']").click()
except Exception as e:
print("Less images found: {}".format(e))
break
html = driver.page_source.split('"')
imges = []
links = []
for i in html:
if i.startswith('https:') and ('gstatic' not in i) and ('google' not in i):
links.append(i.split('"')[0])
for i in html:
if i.startswith('http') and 'usqp=CAU' in i.split('.')[-1]:
imges.append(i.split('"')[0])
for i in html:
if i.startswith('http') and i.split('"')[0].split('.')[-1] in extensions:
imges.append(i.split('"')[0])
links = list(set(links))
imges = list(set(imges))
print(imges)
links_left = Diff(links, imges)
#removing duplicates
urls_new = []
[urls_new.append(x) for x in links_left if x not in urls_new]
file1 = open("page_source.txt", "w", encoding='utf8')
file1.writelines(urls_new)
img_type = []
print("Total images: {}\n".format(len(imges)))
for img in imges:
img_count += 1
print("Downloading image {}:{}".format(img_count, img))
img_type = img.rsplit('.', 1)
try:
req = Request(img, headers=headers)
raw_img = urlopen(req).read()
f = open(download_path+searchtext.replace(" ", "_")+"/" +
str(downloaded_img_count)+"."+"jpeg", "wb")
f.write(raw_img)
f.close
downloaded_img_count += 1
except Exception as e:
print("Download failed: {}".format(e))
finally:
print
if downloaded_img_count >= num_requested:
break
print("Total downloaded: {}/{}".format(downloaded_img_count, img_count))
print("Total images: {}\n".format(len(urls_new)))
for url in urls_new:
img_count = img_scp.img_download(url, download_path+searchtext.replace(" ", "_")+"/", img_count)
driver.quit()
def info(self):
options = EdgeOptions()
options.use_chromium = True
#options.add_argument('--start-maximized')
options.add_argument('--disable-extensions')
driver_path = 'Driver\\msedgedriver.exe'
#Opciones de navegacion
driver = Edge(executable_path=driver_path, options=options)
#inicializamos el navegador
driver.get('https://www.accuweather.com/')
Departamento = "Paysandú"
#COOKIES
WebDriverWait(driver, 10)\
.until(EC.element_to_be_clickable((By.XPATH,
'/html/body/div/div[9]/div/div')))\
.click()
#BUSCADOR
WebDriverWait(driver, 10)\
.until(EC.element_to_be_clickable((By.XPATH,
'/html/body/div/div[1]/div[2]/div[1]/form/input')))\
.send_keys(Departamento)
#CIUDAD
WebDriverWait(driver, 10)\
.until(EC.element_to_be_clickable((By.XPATH,
'/html/body/div/div[1]/div[2]/div[2]/div[2]/div')))\
.click()
#DIAS
WebDriverWait(driver, 10)\
.until(EC.element_to_be_clickable((By.XPATH,
'/html/body/div/div[3]/div/div[3]/a[3]')))\
.click()
card = WebDriverWait(driver, 20)\
.until(EC.frame_to_be_available_and_switch_to_it((By.NAME,
"google_ads_iframe_/6581/web/sam/interstitial/weather/local_home_0")))
if (card):
WebDriverWait(driver, 10)\
.until(EC.element_to_be_clickable((By.XPATH,
"/html/body/div/div/div[1]/div[1]"))).click()
#INFO
WebDriverWait(driver, 10)\
.until(EC.element_to_be_clickable((By.XPATH,
'/html/body/div/div/div[1]/div[1]/div')))
info_clima = driver.find_element_by_xpath(
'/html/body/div/div[5]/div[1]/div[1]')
info_clima = info_clima.text
titulo = driver.find_element_by_css_selector('p.module-title')
titulo = titulo.text
#print(titulo)
#SEPARAR
datos_semana = info_clima.split(titulo)[1].split('\n')[1:36]
driver.quit()
return datos_semana
from msedge.selenium_tools import options
from msedge.selenium_tools import Edge, EdgeOptions
from time import sleep
options = EdgeOptions()
options.use_chromium = True
driver = Edge(options=options)
options.add_experimental_option("excludeSwitches", ['enable-automation'])
kodSekolah = "NEB2054"
driver.get(
"https://docs.google.com/forms/d/e/1FAIpQLSduotkz5L48QN_vXksFCerIfHF4ihDQyxlARbL4BneSeqzNWg/viewform"
)
print("Google sites opened")
sleep(0.5)
KodSekolah_box = driver.find_element_by_class_name(
"quantumWizTextinputPaperinputEl")
KodSekolah_box.send_keys(kodSekolah)
sleep(0.2)
def scrape(secure=False):
options = EdgeOptions()
options.use_chromium = True
driver = Edge(options=options)
query = input("▁ ▂ ▄ ▅ ▆ ▇ █ 𝐄𝐧𝐭𝐞𝐫 𝐭𝐡𝐞 𝐓𝐞𝐱𝐭 𝐭𝐨 𝐬𝐞𝐚𝐫𝐜𝐡 █ ▇ ▆ ▅ ▄ ▂ ▁\n\n ")
print("\n𝘚𝘵𝘢𝘳𝘵𝘦𝘥 𝘚𝘤𝘳𝘢𝘱𝘪𝘯𝘨 ↦↦↦↦↦↦↦↦↦↦")
print("\nPlease Wait ............\n")
driver.get("https://www.twitter.com/login")
driver.maximize_window()
username = driver.find_element_by_xpath(
'//input[@name="session[username_or_email]"]')
username.send_keys("*****@*****.**")
#password=getpass()
userpas = driver.find_element_by_xpath(
'//input[@name="session[password]"]')
userpas.send_keys('-----')
userpas.send_keys(Keys.RETURN)
sleep(2)
if secure:
username = driver.find_element_by_xpath(
'//input[@name="session[username_or_email]"]')
username.send_keys("031-----")
userpas = driver.find_element_by_xpath(
'//input[@name="session[password]"]')
userpas.send_keys('----')
userpas.send_keys(Keys.RETURN)
sleep(2)
search = driver.find_element_by_xpath(
'//input[@aria-label="Search query"]')
search.send_keys('"پاک فوج" lang:ur -filter:links filter:replies')
search.send_keys(Keys.RETURN)
sleep(1.5)
driver.find_element_by_link_text("Latest").click()
data = []
tweet_ids = set()
last_position = driver.execute_script("return window.pageYOffset;")
scrolling = True
while scrolling:
posts = driver.find_elements_by_xpath('//div[@data-testid="tweet"]')
for post in posts[-15:]:
tweet = scrap_tweets(post)
if tweet:
tweet_id = "".join(tweet)
if tweet_id not in tweet_ids:
tweet_ids.add(tweet_id)
data.append(tweet)
scroll_attempt = 0
while True:
driver.execute_script(
"window.scrollTo(0,document.body.scrollHeight);")
sleep(1)
curr_position = driver.execute_script("return window.pageYOffset;")
if last_position == curr_position:
scroll_attempt += 1
if scroll_attempt >= 3:
scrolling = False
break
else:
sleep(2)
else:
last_position = curr_position
break
return data
print("no hay numeros")
else:
if ticket == 99:
print("no hay numero de 7 digitos")
else:
print("el ticket debe ser", ticket)
print(lista)
ticket = lista[0]
print(ticket)
edge_option = EdgeOptions()
edge_option.add_argument("hide_console")
driver = Edge("C:\Google\msedgedriver", service_args=["hide_console"])
url = "http://10.1.27.11:8080/tfs/TFSYPF/E2E/_workitems?_a=edit&id="
urlarmada = url + ticket
driver.get(urlarmada)
driver.implicitly_wait(8)
attachment = "ui-id-7"
driver.find_element_by_id(attachment).click()
direarchivo = dir_inicial + nombre_archivo
driver.implicitly_wait(5)
driver.find_element_by_xpath(
"/html/body/div[4]/div[2]/div/form/input[1]").send_keys(
direarchivo)
driver.implicitly_wait(5)
driver.find_element_by_xpath(
"/html/body/div[4]/div[3]/div/button[1]").click()
driver.implicitly_wait(5)
tweet = (username, handle, postdate, text, emojis, reply_cnt, retweet_cnt, like_cnt)
return tweet
# user
user = input('username: '******'Password: '******'search term: ')
# web driver
options = EdgeOptions()
options.use_chromium = True
driver = Edge(options=options)
# login
driver.get('https://www.twitter.com/login')
driver.maximize_window()
username = driver.find_element_by_xpath('//input[@name="session[username_or_email]"]')
username.send_keys(user)
password = driver.find_element_by_xpath('//input[@name="session[password]"]')
password.send_keys(my_password)
password.send_keys(Keys.RETURN)
sleep(1)
# search
search_input = driver.find_element_by_xpath('//input[@aria-label="Search query"]')
search_input.send_keys(search_term)
search_input.send_keys(Keys.RETURN)
sleep(1)
class TwitterBot():
def __init__(self):
self.driver = Edge()
self.driver.maximize_window()
self.driver.get('https://twitter.com')
self.driver.implicitly_wait(3)
def goToTwitter(self):
self.driver.get('https://twitter.com')
def login(self):
self.driver.find_element_by_xpath("//a[@href='/login']").click()
#I used sleep because before this time there is another instance of an element named like below.
#It is crucial to get the right element in order to interact with it.
sleep(1)
self.driver.find_element_by_xpath(
"//input[@name='session[username_or_email]']").send_keys(username)
self.driver.find_element_by_xpath(
"//input[@name='session[password]']").send_keys(password)
self.driver.find_element_by_xpath(
"//div[@data-testid='LoginForm_Login_Button']").click()
def basicSearch(self, topic):
self.driver.find_element_by_xpath(
"//input[@data-testid='SearchBox_Search_Input']").send_keys(topic)
self.driver.find_element_by_xpath(
"//input[@data-testid='SearchBox_Search_Input']").submit()
def advancedSearch(self, exact, any, none, hashtags, dateFrom, dateTo):
finalSearch = ''
#This is to accommodate for different search types that a user might want.
if exact != None:
finalSearch += '"' + exact + '" '
if any != None:
finalSearch += '(' + any + ') '
if none != None:
finalSearch += '-' + none + ' '
if hashtags != None:
finalSearch += '(#' + hashtags + ') '
if dateTo != None:
finalSearch += 'until:' + dateTo + ' '
if dateFrom != None:
finalSearch += 'since:' + dateFrom + ' '
self.driver.find_element_by_xpath(
"//input[@data-testid='SearchBox_Search_Input']").send_keys(
finalSearch)
self.driver.find_element_by_xpath(
"//input[@data-testid='SearchBox_Search_Input']").submit()
def scrapeTweets(self, desiredNum):
allLines = ''
oldDataLines = []
dataLines = ['init']
tweetsFile = open('tweets.csv', 'w')
#I included this array to help clean data later
dirtyArray = [
'Quote Tweet', 'Promoted', 'Show this thread', '', '\n', ' '
]
numDataLines = 0
while numDataLines < desiredNum and oldDataLines != dataLines:
oldDataLines = dataLines
sleep(1)
#all these are different types of data that I do not want to pick up.
dirtyData = self.driver.find_elements_by_xpath(
"//div[@class='css-1dbjc4n r-1d09ksm r-18u37iz r-1wbh5a2']")
dirtyData2 = self.driver.find_elements_by_xpath(
"//div[@class = 'css-1dbjc4n r-18u37iz r-1wtj0ep r-156q2ks r-1mdbhws']"
)
dirtyData3 = self.driver.find_elements_by_xpath(
"//div[contains(text(),'Replying to')]")
dirtyData4 = self.driver.find_elements_by_xpath(
"//div[@role = 'blockquote']")
#adding all the dirty data into one array
for dirt in dirtyData2:
dirtyData.append(dirt)
for dirt in dirtyData3:
dirtyData.append(dirt)
for dirt in dirtyData4:
dirtyData.append(dirt)
#the data is stored with strings with many lines so I split the strings up by line and have an array where each index is one lin
dirtyLines = []
for dirt in dirtyData:
dirt = dirt.text
chunks = dirt.split('\n')
for chunk in chunks:
dirtyLines.append(chunk)
#this includes dirty data that will be weeded out later
data = self.driver.find_elements_by_xpath(
"//div[@data-testid='tweet']")
#same thing I did with dirtyLines
dataLines = []
for datapoint in data:
datapoint = datapoint.text
chunks = datapoint.split('\n')
for chunk in chunks:
dataLines.append(chunk)
#I check oldDataLines as well to avoid redundancy
for line in dataLines:
if line not in dirtyLines and line not in oldDataLines and line not in dirtyArray:
if numDataLines >= desiredNum:
break
try:
noPunctuationLine = re.sub(r'[^\w\s]', '', line)
tweetsFile.write(noPunctuationLine)
tweetsFile.write("\n")
allLines += line
numDataLines += 1
except Exception:
print('This data point not encodable.')
height = self.driver.execute_script(
"return document.documentElement.scrollHeight")
self.driver.execute_script("window.scrollTo(0, " + str(height) +
");")
tweetsFile.close()
return allLines
import time
import pandas as pd
#from selenium.webdriver.chrome.options import Options
options = EdgeOptions()
options.use_chromium = True
#options.add_argument('--start-maximized')
options.add_argument('--disable-extensions')
driver_path = 'Driver\\msedgedriver.exe'
#Opciones de navegacion
driver = Edge(executable_path=driver_path, options=options)
#inicializamos el navegador
driver.get('https://www.accuweather.com/')
Departamento = "Paysandú"
#COOKIES
WebDriverWait(driver, 10)\
.until(EC.element_to_be_clickable((By.XPATH,
'/html/body/div/div[9]/div/div')))\
.click()
#BUSCADOR
WebDriverWait(driver, 10)\
.until(EC.element_to_be_clickable((By.XPATH,
'/html/body/div/div[1]/div[2]/div[1]/form/input')))\
.send_keys(Departamento)
#CIUDAD
class Session:
def __init__(self, username, password, sleep_time=2):
self.username = username
self.password = password
self.sleep_time = sleep_time
options = EdgeOptions()
options.use_chromium = True
self.driver = Edge(options=options)
def login(self):
self.driver.get("https://www.twitter.com/login")
sleep(self.sleep_time)
u_name = self.driver.find_element_by_xpath(
'//input[@name="session[username_or_email]"]')
u_name.send_keys(self.username)
p_word = self.driver.find_element_by_xpath(
'//input[@name="session[password]"]')
p_word.send_keys(self.password)
p_word.send_keys(Keys.RETURN)
sleep(self.sleep_time)
def tweet_selection(self, search_str, csv_tit, max_tweets=300):
sleep(self.sleep_time)
search_input = self.driver.find_element_by_xpath(
'//input[@aria-label="Search query"]')
search_input.clear()
search_input.send_keys(search_str)
search_input.send_keys(Keys.RETURN)
sleep(self.sleep_time)
data = []
tweet_ids = set()
last_pos = self.driver.execute_script("return window.pageYOffset;")
scrolling = True
while scrolling:
cards = self.driver.find_elements_by_xpath(
'//div[@data-testid="tweet"]')
for card in cards[-15:]:
tweet = self.get_tweet_data(card)
if tweet:
tweet_id = ''.join(tweet)
if tweet_id not in tweet_ids:
tweet_ids.add(tweet_id)
data.append(tweet)
scroll_attempt = 0
while True:
self.driver.execute_script(
'window.scrollTo(0, document.body.scrollHeight);')
sleep(self.sleep_time)
curr_pos = self.driver.execute_script(
"return window.pageYOffset;")
if last_pos == curr_pos:
scroll_attempt += 1
if scroll_attempt >= 3:
scrolling = False
break
else:
sleep(2 * self.sleep_time)
else:
last_pos = curr_pos
break
with open(csv_tit, 'w', encoding="utf-8") as out:
csv_out = csv.writer(out)
csv_out.writerow([
'user', 'date', 'text', 'quoting', 'reply count',
'retweet count', 'like count'
])
for row in data:
csv_out.writerow(row)
def get_tweet_data(self, card):
user = card.find_element_by_xpath('.//span[contains(text(),"@")]').text
try:
date = card.find_element_by_xpath('.//time').get_attribute(
'datetime')
except NoSuchElementException:
return
text = card.find_element_by_xpath('.//div[2]/div[2]/div[1]').text
responding = card.find_element_by_xpath('.//div[2]/div[2]/div[2]').text
reply_count = card.find_element_by_xpath(
'.//div[@data-testid="reply"]').text
retweet_count = card.find_element_by_xpath(
'.//div[@data-testid="retweet"]').text
like_count = card.find_element_by_xpath(
'.//div[@data-testid="like"]').text
tweet = (user, date, text, responding, reply_count, retweet_count,
like_count)
return tweet
def tweet(self, tuit): # REQUIERE INTERACTUAR CON EDGE
sleep(self.sleep_time)
tuit_input = self.driver.find_element_by_xpath(
'//div[@data-testid="tweetTextarea_0"]')
tuit_input.clear()
tuit_input.send_keys(tuit)
def download(url):
options = EdgeOptions()
options.use_chromium =True
# option = webdriver.ChromeOptions()
# option.add_argument('headless')
options.add_argument('log-level=3')
driver = Edge(options=options)
# driver = webdriver.Chrome(
# executable_path='.//chromedriver', chrome_options=option)
title = "output"
try:
driver.set_page_load_timeout(15)
driver.get(url)
title = driver.title
except:
print("Timeout - start download anyway.")
print(f'道客巴巴: 《{title}》')
time.sleep(5)
try:
# 展开全部
elem_cont_button = driver.find_element_by_id("continueButton")
driver.execute_script(
"arguments[0].scrollIntoView(true);", elem_cont_button)
actions = ActionChains(driver)
actions.move_to_element(elem_cont_button).perform()
time.sleep(0.5)
elem_cont_button.click()
except NoSuchElementException:
pass
# 获取页数
num_of_pages = driver.find_element_by_id('readshop').find_element_by_class_name(
'mainpart').find_element_by_class_name('shop3').find_element_by_class_name('text').get_attribute('innerHTML')
num_of_pages = int(num_of_pages.split(' ')[-1])
for i in range(5):
# 缩放
driver.find_element_by_id('zoomInButton').click()
time.sleep(0.5)
if os.path.exists(f'./temp/{title}'):
shutil.rmtree(f'./temp/{title}')
os.makedirs(f'./temp/{title}')
for pages in trange(num_of_pages):
time.sleep(0.5)
canvas_id = "page_" + str(pages + 1)
pagepb_id = "pagepb_" + str(pages + 1)
element = driver.find_element_by_id(canvas_id)
driver.execute_script("arguments[0].scrollIntoView(true);", element)
actions = ActionChains(driver)
actions.move_to_element(element).perform()
time.sleep(0.5)
# Check loading status
while(len(driver.find_element_by_id(pagepb_id).get_attribute('innerHTML')) != 0):
time.sleep(1)
# print(driver.find_element_by_id(
# pagepb_id).get_attribute('innerHTML'))
js_cmd = "var canvas = document.getElementById('{}');".format(canvas_id) + \
"return canvas.toDataURL();"
img_data = driver.execute_script(js_cmd)
img_data = (img_data[22:]).encode()
with open(f"./temp/{title}/{pages}.png", "wb") as fh:
fh.write(base64.decodebytes(img_data))
driver.quit()
print('下载完毕，正在转码')
conpdf(f'output/{title}.pdf', f'temp/{title}', '.png')
class Web_scraping:
def __init__(self):
'''Initialize the application'''
#As using the standard webdriver was giving warnings and messing up the terminal, I used the code below to show just what I want.
self.opt = EdgeOptions()
self.opt.add_experimental_option('excludeSwitches', ['enable-logging'])
self.opt.add_argument("--start-maximized")
self.opt.use_chromium = True
self.driver = Edge(
executable_path=
r"C:\\Program Files (x86)\\Microsoft\\Edge\\Application\\msedgedriver.exe",
options=self.opt)
def games_link(self):
'''Create a list with all season event's link and then create another list with all event's link'''
#Creating list with the all season's link
self.season_pages_list = []
for y in range(2008, 2022):
#Creating the seasons links as str and adding it to a list
self.season_link = 'https://www.worldsurfleague.com/events/' + str(
y) + '/mct?all=1'
self.season_pages_list.append(self.season_link)
#Creating a list with the all event's link from each season
self.events_link_list = []
for link in self.season_pages_list:
self.driver.get(link)
#Getting all the events links as selenium format
self.event_links = self.driver.find_elements_by_xpath(
'//a[@class="event-schedule-details__event-name"]')
#Finding the class status completed is needed once it's possible to stop the process on it.
self.event_status = self.driver.find_elements_by_xpath(
'//span[@class="event-status event-status--completed"]')
#Creating event's link list
for i in range(0, len(self.event_status)):
#Getting the links for each event as a str format
self.link_attribute = self.event_links[i].get_attribute('href')
self.events_link_list.append(self.link_attribute)
with open('events.txt', 'w') as f:
for item in self.events_link_list:
f.write("%s\n" % item)
print('FINISHED')
#Getting data inside which event
def event_stats(self):
#TXT file with all events link to list
self.events_link = [
line[0]
for line in pd.read_fwf('events.txt', header=None).values.tolist()
]
#for link in self.events_link:
self.driver.get(self.events_link[0])
#list of all heats
self.all_heats_lists = []
while True:
#Gets all the waves scores, athletes, nationalities and heats on the page as list.
self.waves = self.driver.find_elements_by_xpath(
'//*[@class="score"]')
self.athletes = self.driver.find_elements_by_xpath(
'//*[@class="athlete-name"]')
self.nationalities = self.driver.find_elements_by_xpath(
'//*[@class="athlete-country-flag"]')
self.heat = self.driver.find_elements_by_xpath(
'//*[@class="new-heat-hd-name"]')
#Gets the round name
self.round = self.driver.find_elements_by_xpath(
'//*[@class="carousel-item is-selected"]')
if len(self.round) == 0:
self.round = self.driver.find_elements_by_xpath(
'//*[@class="carousel-item last is-selected"]')
#Gets the number of surfers and heats on the round, such as the avg surfers per heat (must be 2 or 3)
self.number_of_surfers = int(len(self.waves) / 18)
#As the final round only has 1 heat, the find_element_by_class_name gets a 'WebDriver' element and not a list
self.number_of_heats = len(self.heat)
self.surfers_per_heat = int(self.number_of_surfers /
self.number_of_heats)
#there's a count to deduct 1 stage and gets the round name for each round.
self.count = 0
#Gets the stats for each heat
self.heat_data = []
for g in range(0, self.number_of_heats):
#Page stats
#Event stats
self.event_turn = self.driver.find_element_by_class_name(
'event-meta-tour-info').text.split()[2][1:]
self.event_period = self.driver.find_element_by_class_name(
'event-schedule__date-range').text
self.event_name = self.driver.find_element_by_class_name(
'event-title').text.split('\n')[0]
self.event_local = re.split(
r'(\d+)',
self.driver.find_element_by_class_name(
'event-meta-tour-info').text)[2]
self.avg_wave_score = re.split(
r'(\d+\.\d+)',
self.driver.find_element_by_class_name(
'new-heat-hd-status').text)[1]
#Heat's id for the database
self.heat_id = (f'heat{g + 1}' + self.round[0].text +
self.event_turn +
self.event_period[-4:]).lower()
#Surfer stats
self.surfer1 = self.athletes[g * 2].text
self.surfer1_nat = self.nationalities[g *
2].get_attribute('title')
self.surfer1_best_w1 = self.waves[g * 18 + (1 - 1)].text
self.surfer1_best_w2 = self.waves[g * 18 + (2 - 1)].text
self.surfer1_total = self.waves[g * 18 + (3 - 1)].text
self.surfer1_w01 = self.waves[g * 18 + (4 - 1)].text
self.surfer1_w02 = self.waves[g * 18 + (5 - 1)].text
self.surfer1_w03 = self.waves[g * 18 + (6 - 1)].text
self.surfer1_w04 = self.waves[g * 18 + (7 - 1)].text
self.surfer1_w05 = self.waves[g * 18 + (8 - 1)].text
self.surfer1_w06 = self.waves[g * 18 + (9 - 1)].text
self.surfer1_w07 = self.waves[g * 18 + (10 - 1)].text
self.surfer1_w08 = self.waves[g * 18 + (11 - 1)].text
self.surfer1_w09 = self.waves[g * 18 + (12 - 1)].text
self.surfer1_w10 = self.waves[g * 18 + (13 - 1)].text
self.surfer1_w11 = self.waves[g * 18 + (14 - 1)].text
self.surfer1_w12 = self.waves[g * 18 + (15 - 1)].text
self.surfer1_w13 = self.waves[g * 18 + (16 - 1)].text
self.surfer1_w14 = self.waves[g * 18 + (17 - 1)].text
self.surfer1_w15 = self.waves[g * 18 + (18 - 1)].text
#Surfer 2 stats
self.surfer2 = self.athletes[g * 2 + 1].text
self.surfer2_nat = self.nationalities[g * 2 +
1].get_attribute('title')
self.surfer2_best_w1 = self.waves[g * 18 + (19 - 1)].text
self.surfer2_best_w2 = self.waves[g * 18 + (20 - 1)].text
self.surfer2_total = self.waves[g * 18 + (21 - 1)].text
self.surfer2_w01 = self.waves[g * 18 + (22 - 1)].text
self.surfer2_w02 = self.waves[g * 18 + (23 - 1)].text
self.surfer2_w03 = self.waves[g * 18 + (24 - 1)].text
self.surfer2_w04 = self.waves[g * 18 + (25 - 1)].text
self.surfer2_w05 = self.waves[g * 18 + (26 - 1)].text
self.surfer2_w06 = self.waves[g * 18 + (27 - 1)].text
self.surfer2_w07 = self.waves[g * 18 + (28 - 1)].text
self.surfer2_w08 = self.waves[g * 18 + (29 - 1)].text
self.surfer2_w09 = self.waves[g * 18 + (30 - 1)].text
self.surfer2_w10 = self.waves[g * 18 + (31 - 1)].text
self.surfer2_w11 = self.waves[g * 18 + (32 - 1)].text
self.surfer2_w12 = self.waves[g * 18 + (33 - 1)].text
self.surfer2_w13 = self.waves[g * 18 + (34 - 1)].text
self.surfer2_w14 = self.waves[g * 18 + (35 - 1)].text
self.surfer2_w15 = self.waves[g * 18 + (36 - 1)].text
#Inputing all variables into the heat_data list
self.heat_data.append(self.heat_id)
self.heat_data.append(self.event_name)
self.heat_data.append(self.event_local)
self.heat_data.append(self.event_turn)
self.heat_data.append(self.event_period)
self.heat_data.append(self.avg_wave_score)
self.heat_data.append(self.surfer1)
self.heat_data.append(self.surfer1_nat)
self.heat_data.append(self.surfer1_best_w1)
self.heat_data.append(self.surfer1_best_w2)
self.heat_data.append(self.surfer1_total)
self.heat_data.append(self.surfer1_w01)
self.heat_data.append(self.surfer1_w02)
self.heat_data.append(self.surfer1_w03)
self.heat_data.append(self.surfer1_w04)
self.heat_data.append(self.surfer1_w05)
self.heat_data.append(self.surfer1_w06)
self.heat_data.append(self.surfer1_w07)
self.heat_data.append(self.surfer1_w08)
self.heat_data.append(self.surfer1_w09)
self.heat_data.append(self.surfer1_w10)
self.heat_data.append(self.surfer1_w11)
self.heat_data.append(self.surfer1_w12)
self.heat_data.append(self.surfer1_w13)
self.heat_data.append(self.surfer1_w14)
self.heat_data.append(self.surfer1_w15)
self.heat_data.append(self.surfer2)
self.heat_data.append(self.surfer2_nat)
self.heat_data.append(self.surfer2_best_w1)
self.heat_data.append(self.surfer2_best_w2)
self.heat_data.append(self.surfer2_total)
self.heat_data.append(self.surfer2_w01)
self.heat_data.append(self.surfer2_w02)
self.heat_data.append(self.surfer2_w03)
self.heat_data.append(self.surfer2_w04)
self.heat_data.append(self.surfer2_w05)
self.heat_data.append(self.surfer2_w06)
self.heat_data.append(self.surfer2_w07)
self.heat_data.append(self.surfer2_w08)
self.heat_data.append(self.surfer2_w09)
self.heat_data.append(self.surfer2_w10)
self.heat_data.append(self.surfer2_w11)
self.heat_data.append(self.surfer2_w12)
self.heat_data.append(self.surfer2_w13)
self.heat_data.append(self.surfer2_w14)
self.heat_data.append(self.surfer2_w15)
self.all_heats_lists.append(self.heat_data.copy())
self.heat_data.clear()
#Click on the previous round botton
print(self.all_heats_lists)
try:
self.prev_round_bt = self.driver.find_element_by_xpath(
'//*[@class="flickity-button-icon"]').click()
except:
self.prev_round_bt = self.driver.find_element_by_xpath(
'//*[@class="flickity-button-icon"]')
self.driver.execute_script("arguments[0].scrollIntoView();",
self.prev_round_bt)
time.sleep(.5)
self.prev_round_bt.click()
time.sleep(2.5)
def run():
email = read_decrypt(efile)
password = read_decrypt(pfile)
cemail = str(email)
cpassword = str(password)
with open("browserdetails.txt", "r") as f:
data = [line.rstrip('\n') for line in f]
browser = data[0].lower()
gpu = data[1].lower()
if browser == 'edge':
try:
requests.get("http://www.google.com")
print('Connection Established.')
l1 = tk.Label(top, text=" Connection Established. ", bg='white')
l1.config(font=('helvetica', 15, "bold"))
canvas.create_window(200, 410, window=l1)
options = EdgeOptions()
options.use_chromium = True
options.add_argument("--start-maximized")
if gpu == 'no':
options.add_argument("window-size=1920,1080")
options.add_argument("--headless")
options.add_argument("disable-gpu")
options.add_argument("-inprivate")
driver = Edge(resource_path('msedgedriver.exe'), options=options)
driver.get('https://gokano.com/')
try:
email = WebDriverWait(driver, 3).until(
EC.presence_of_element_located((By.NAME, 'email')))
print("Page is ready!")
l1 = tk.Label(top,
text=" Page is Ready. ",
bg='white')
l1.config(font=('helvetica', 15, "bold"))
canvas.create_window(200, 410, window=l1)
email.send_keys(cemail)
password = driver.find_element_by_name('password')
password.send_keys(cpassword)
time.sleep(3)
button = driver.find_element_by_class_name('submit')
button.click()
try:
driver.find_element_by_class_name('gokan-alert-error')
print("Invalid Credintials")
l1 = tk.Label(top,
text=" Invalid Credintials. ",
bg='white')
l1.config(font=('helvetica', 15, "bold"))
canvas.create_window(200, 410, window=l1)
time.sleep(3)
driver.quit()
except NoSuchElementException:
print('Login sucessful')
l1 = tk.Label(top, text=" Login Successful. ", bg='white')
l1.config(font=('helvetica', 15, "bold"))
canvas.create_window(200, 410, window=l1)
except TimeoutException:
print("Login Error!")
l1 = tk.Label(top, text=" Login Error! ", bg='white')
l1.config(font=('helvetica', 15, "bold"))
canvas.create_window(200, 410, window=l1)
# [email protected]
time.sleep(3)
driver.quit()
time.sleep(3)
try:
cdp = driver.find_element_by_link_text('Collect daily points')
cdp.click()
write_time()
time.sleep(3)
driver.quit()
except NoSuchElementException:
print('Already collected')
l1 = tk.Label(top,
text=" Points Already Collected. ",
bg='white')
l1.config(font=('helvetica', 15, "bold"))
canvas.create_window(200, 410, window=l1)
time.sleep(3)
driver.quit()
print('Automation completed')
l1 = tk.Label(top, text=" Automation Completed. ", bg='white')
l1.config(font=('helvetica', 15, "bold"))
canvas.create_window(200, 410, window=l1)
time.sleep(3)
write_time()
driver.quit()
except requests.ConnectionError:
print('Could not connect')
l1 = tk.Label(top, text=" Couldn't Connect. ", bg='white')
l1.config(font=('helvetica', 15, "bold"))
canvas.create_window(200, 410, window=l1)
driver.quit()
from msedge.selenium_tools import Edge, EdgeOptions
from selenium.webdriver.common.keys import Keys
import time
import random
options = EdgeOptions()
options.use_chromium = True
word_list=["stack overflow how to capture traffic", "java display thread", "p2p network"]
driver = Edge(options=options)
driver.get("https://www.google.com/")
driver.maximize_window()
time.sleep(1)
inputElement = driver.find_element_by_id("zV9nZe")
inputElement.click()
inputElement = driver.find_element_by_xpath("/html/body/div[1]/div[3]/form/div[1]/div[1]/div[1]/div/div[2]/input")
inputElement.send_keys(random.choice(word_list), Keys.ENTER)
element = driver.find_element_by_xpath("/html/body/div[7]/div/div[9]/div[1]/div/div[2]/div[2]/div/div/div/div[1]/div/div[1]/a")
time.sleep(1)
driver.get(element.get_attribute('href'))
time.sleep(1)
lenOfPage = driver.execute_script("window.scrollTo(0, document.body.scrollHeight);var lenOfPage=document.body.scrollHeight;return lenOfPage;")
time.sleep(4)
driver.close()
from msedge.selenium_tools import Edge, EdgeOptions
from selenium.webdriver.common.keys import Keys
import time
import random
options = EdgeOptions()
options.use_chromium = True
word_list = [
"https://pt.wikipedia.org/wiki/Border_Gateway_Protocol",
"https://pt.wikipedia.org/wiki/Multi_Protocol_Label_Switching",
"https://pt.wikipedia.org/wiki/Open_Shortest_Path_First"
]
driver = Edge(options=options)
driver.get("https://pt.wikipedia.org/")
driver.maximize_window()
time.sleep(5)
driver.get(random.choice(word_list))
time.sleep(4)
driver.execute_script("window.scrollTo(0, 300)")
time.sleep(4)
driver.execute_script("window.scrollTo(300, 600)")
time.sleep(4)
driver.execute_script("window.scrollTo(600, 900)")
time.sleep(4)
driver.execute_script("window.scrollTo(900, 1200)")
time.sleep(4)
driver.execute_script("window.scrollTo(1200, 1500)")
time.sleep(4)
driver.close()
after=time.perf_counter()
pingms=after-before
pingms=round(pingms,2)+1
except:
pingms=2
print("-Calculated action delay: "+str(pingms))
# Maximize window for no real reason
browser.maximize_window()
for i in range(int(nb)):
passed=passed+1
try:
# Open browser.
if firstTime == True:
browser.get("https://kahoot.it/")
else:
browser.execute_script("window.open('');")
browser.switch_to.window(browser.window_handles[total])
browser.get("https://kahoot.it/")
#if firstTime == False:
# wait=WebDriverWait(browser, 3)
# alert=wait.until(EC.alert_is_present())
# alert.accept()
time.sleep((pingms/2))
# Find game id element and enter game code.
search=browser.find_element_by_name("gameId")
search.click()
search.send_keys(qp)
search.send_keys(Keys.RETURN)
print("-Joined Game")
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from msedge.selenium_tools import Edge, EdgeOptions
work_root = Path(os.path.dirname(os.path.abspath(__file__))).parent
browser_driver = os.path.join(work_root, 'drivers\\msedgedriver.exe')
print(browser_driver)
options = EdgeOptions()
options.use_chromium = True
driver = Edge(executable_path=browser_driver, options=options)
driver.get(
r"https://accounts.spotify.com/en/login?continue=https:%2F%2Fpodcasters.spotify.com%2Fgateway"
)
WebDriverWait(driver, 60).until(
EC.presence_of_element_located(
(By.XPATH, '//body[contains(@data-qa, "s4-podcasters")]')))
print("Logged in!")
cookie_container = {
"sp_dc": driver.get_cookie("sp_dc")["value"],
"sp_key": driver.get_cookie("sp_key")["value"],
}
with open('_sc.json', 'w') as cookie_file:
json.dump(cookie_container, cookie_file)
def _init_browser(url: str, str_cookie=None):
global browser
if browser:
# Browser is already initilized
return browser
# Initilize the browser
if (sys.platform == "win32"):
# If on Windows
try:
# Try the Chrome webdriver
options = ChromeOptions()
# Add options for imporved performance
options.add_argument("--disable-extensions")
options.add_argument("--disable-gpu")
options.add_argument("--no-sandbox")
# Run in headless mode
options.add_argument("--headless")
# Hide log message
options.add_experimental_option('excludeSwitches',
['enable-logging'])
# Get the webdriver
browser = webdriver.Chrome(
executable_path="webdrivers/chromedriver.exe", options=options)
# Test if Chrome binary exist
browser.get(url)
log.debug("initialized Chrome Driver")
except:
log.debug("Failed to set-up Chrome webdriver")
# Else, try the MS Edge webdriver
try:
edge_options = EdgeOptions()
edge_options.use_chromium = True
# Run in headless mode
edge_options.add_argument('headless')
edge_options.add_argument('disable-gpu')
# Get the webdriver
browser = Edge(executable_path="webdrivers/msedgedriver.exe",
options=edge_options)
log.debug("initialized Edge Driver")
except:
log.error("Could not set-up a webdriver")
log.debug(
"Chrome or Chromium Edge must be installed to scan for DOM-based XSS vulnerability"
)
return None
else:
# *nix
try:
options = FirefoxOptions()
# Run in headless mode
options.add_argument("--headless")
# Get the webdriver
try:
browser = webdriver.Firefox(options=options)
except:
log.debug("geckodriver is not in PATH")
browser = webdriver.Firefox(
executable_path="webdrivers/geckodriver", options=options)
# Test if Firefox binary exist
browser.get(url)
log.debug("initialized Firefox Driver")
except:
log.error("Could not set-up a webdriver")
log.exception(
"Firefox must be installed to scan for DOM-based XSS vulnerability"
)
return None
if str_cookie:
cookie = SimpleCookie()
cookie.load(str_cookie)
cookies = {}
for key, morsel in cookie.items():
cookies[key] = morsel.value
browser.get(url)
# browser.delete_all_cookies()
for key in cookies.keys():
value = cookies[key]
browser.add_cookie({"name": key, "value": value})
return browser
from msedge.selenium_tools import Edge, EdgeOptions
# Launch Microsoft Edge (Chromium)
options = EdgeOptions()
options.use_chromium = True
driver = Edge(options=options)
driver.get("https://github.com/netteNz")
raise Exception("WHAT SYSTEM ARE YOU USING?!")
def version_check():
if sys.version_info < (3, 4):
raise Exception("You must use Python 3.4 or higher.")
if __name__ == "__main__":
version_check()
edgeOptions = EdgeOptions()
edgeOptions.use_chromium = True
edgeOptions.add_argument("user-data-dir=" + get_profile())
edgeOptions.add_argument("disable-extensions")
edgeOptions.add_argument("disable-gpu")
edgeOptions.add_argument("headless")
driver = Edge(options=edgeOptions)
#driver.implicitly_wait(10)
try:
driver.get("https://google.com")
#TODO: YOUR CODE HERE
except NoSuchElementException as e:
print("ERROR: Something went wrong!")
print(e)
finally:
die()
import time
# Configure Selenium
from msedge.selenium_tools import Edge, EdgeOptions, EdgeService
MSEDGEDRIVER_PATH= "C:\\edgedriver_win64\\msedgedriver.exe"
if __name__ == "__main__":
service = EdgeService(MSEDGEDRIVER_PATH)
options = EdgeOptions()
options.use_chromium = True
driver = Edge(executable_path=MSEDGEDRIVER_PATH,options=options)
# Synthetic test
BASE_URL = "http://www.yourapp.com"
driver.get(BASE_URL+"/ims.html")
time.sleep(3)
try:
element = driver.find_element_by_id("Home")
element.click()
time.sleep(3)
element = driver.find_element_by_id("Orders")
element.click()
time.sleep(3)
element = driver.find_element_by_id("Portfolio")
element.click()
time.sleep(3)
from msedge.selenium_tools import Edge, EdgeOptions
import time, pyautogui as pya, keyboard, random
webdriver_location="msedgedriver.exe"
options=EdgeOptions()
options.use_chromium=True
options.binary_location=r'C:\Program Files (x86)\Microsoft\Edge\Application\msedge.exe'
browser=Edge(options=options,executable_path=webdriver_location)
running=False
keybind=input("\n\n\n[!!] -Start typeing keybind, press on start of test: ")
while True:
try:
if keyboard.is_pressed(keybind) == True and running == False:
browser.get("https://www.typing.com/student/typing-test/1-minute")
runningEFE=True
newtype=""
typestring=[]
print('-Reading...')
elements=browser.find_elements_by_class_name('screenBasic-letter')
print("-Typing...")
for e in elements:
text=e.get_attribute("textContent")
print(text)
if "\xa0" in text:
text=str(text).replace("\xa0"," ")
if "\n" in text:
text=str(text).replace("\n"," ")
typestring.append(text)
for c in typestring:
