def get_car(driver: webdriver, i: int) -> None:
try:
#time.sleep(10)
wait: WebDriverWait = WebDriverWait(driver, 10)
element = wait.until(
EC.element_to_be_clickable(
(By.XPATH,
'//*[@id="tbl_list_p"]/tbody/tr[' + str(i) + ']/td[3]/a')))
element.click()
tabs = driver.window_handles
driver.switch_to.window(tabs[1])
car: Dict[str] = {}
car['description'] = driver.find_element_by_css_selector(
'div.search-v>table.tableV').get_attribute('outerHTML')
car['description'] += driver.find_element_by_xpath(
'//*[@id="container"]/div[2]/div[3]/h2[2]').get_attribute(
'outerHTML')
car['description'] += driver.find_element_by_xpath(
'//*[@id="container"]/div[2]/div[3]/table[2]').get_attribute(
'outerHTML')
#car['description'] += driver.find_element_by_xpath('//*[@id="container"]/div[2]/div[3]/h2[3]').get_attribute('outerHTML').rstrip("\n")
#car['description'] += driver.find_element_by_xpath('//*[@id="container"]/div[2]/div[3]/div[2]').get_attribute('outerHTML').rstrip("\n")
car['description'] = rm_new_line(car['description']).encode("utf-8")
car['images'] = get_img_str(driver)
car['count'] = '-1'
car['activation'] = '1'
car['currency'] = 'USD'
download_images(get_img(driver))
driver.close()
driver.switch_to.window(tabs[0])
return car
except Exception as e:
print('Can\'t find element. Reason: %s' % e)
def symbol_to_frames(symbol: str, browser_object: webdriver) -> DataFrame:
""" Takes in the symbol and returns floorsheet dataframe """
search_url: str = f"{BASE_URL}{symbol}"
browser_object.get(search_url)
browser_object.implicitly_wait(IMP_DELAY)
# Click on the nav button
browser_object.find_element_by_id("navFloorSheet").click()
wait()
intent_block = browser_object.find_element_by_id(
"ctl00_ContentPlaceHolder1_CompanyDetail1_divDataFloorsheet")
frame_block = block_to_frame(intent_block)
next_button = browser_object.find_element_by_xpath(
"//a[@title='Next Page']")
pages = []
while get_pages(intent_block)["current_page"] <= get_pages(
intent_block)["last_page"] and get_pages(
intent_block)["current_page"] not in pages:
pages.append(int(get_pages(intent_block)["current_page"]))
next_button.click()
wait()
next_button = browser_object.find_element_by_xpath(
"//a[@title='Next Page']")
intent_block = browser_object.find_element_by_id(
"ctl00_ContentPlaceHolder1_CompanyDetail1_divDataFloorsheet")
inter_frame = block_to_frame(intent_block)
frame_block = pd.concat([frame_block, inter_frame], axis=0)
pages = []
return frame_block
def __call__(self, driver: webdriver):
try:
# Return true if our original spin button is back
if self.spin_element.is_displayed():
return True
# If a different button is visible, click it.
for element in self.other_elements:
try:
if element.is_displayed():
try:
element.click()
except (slex.ElementNotVisibleException,
slex.WebDriverException):
continue
else:
break
except slex.NoSuchElementException:
continue
# Some games explicitly pass an "Insufficient funds" dialog.
# Return True if this happens...
try:
driver.find_element_by_xpath(self.insufficient_xpath_visible)
return True
except slex.NoSuchElementException:
pass
except slex.StaleElementReferenceException:
return False
def _neova_helper(self, driver: webdriver) -> None:
"""[summary]
"""
# farligt med att namnge filerna utifrpn ordningen i listan på webben. denna kan ändras. Bättre om metadata fanns i csvfilen.
neova_anläggningar: Dict = {
"Mimer_7": '//*[@id="app"]/div/div/div[1]/div[2]/div[1]/nav/div/div[2]/div/ul/li[2]/div[2]',
"Läroverket_Stålhagskolan": '//*[@id="app"]/div/div/div[1]/div[2]/div[1]/nav/div/div[2]/div/ul/li[3]/div[2]',
"Smeden_8": '//*[@id="app"]/div/div/div[1]/div[2]/div[1]/nav/div/div[2]/div/ul/li[4]/div[2]',
"Solvändan_2": '//*[@id="app"]/div/div/div[1]/div[2]/div[1]/nav/div/div[2]/div/ul/li[5]/div[1]',
"Läroverket_C_huset": '//*[@id="app"]/div/div/div[1]/div[2]/div[1]/nav/div/div[2]/div/ul/li[6]/div[2]',
}
for key, value in neova_anläggningar.items():
driver.find_element_by_class_name("usage-place-bar__button").click()
ts(5)
driver.find_element_by_xpath(value).click()
ts(5)
driver.find_element_by_class_name("usage-place-menu__close").click()
ts(2)
# ladda ner csv
driver.find_element_by_xpath(
'//*[@id="base__content"]/div[2]/div[1]/div[7]/a'
).click()
ts(5)
self._rename_file(facility=key)
def download_courses(username: str, password: str, download_path: str,
driver: webdriver):
wait30: ui.WebDriverWait = ui.WebDriverWait(driver, 30)
print("LOGGING IN", flush=True)
# Request the login page and wait for it to load
driver.get("https://members.usegolang.com/login")
wait30.until(lambda driver: driver.find_element_by_xpath(
r"""//input[@id="email"]""") is not None)
# Fill in the email and password fields, then hit enter to login
driver.find_element_by_id("email").send_keys(username)
driver.find_element_by_id("password").send_keys(password + "\n")
# Wait until login finishes
wait30.until(lambda driver: driver.find_element_by_xpath(
r"""//small[contains(text(), "Access all of your course contents belows")]"""
) is not None)
print("LOGGED IN", flush=True)
print("DOWNLOADING MAIN LESSONS", flush=True)
# Download all of the main lessons
download_videos("https://members.usegolang.com/twg/lessons/lesson-{}",
download_path, 1, 83, driver)
# Download all of the project lessons
download_videos("https://members.usegolang.com/twg/projects/lesson-{}",
download_path, 1, 19, driver)
def authenticate(cls, driver: webdriver, username: str, password: str,
duo_bypass: str):
"""Goes through Brown Auth process with provided username and password"""
# Check if credentials are provided
if not (username and password and duo_bypass):
raise Exception(f"Credentials incorrectly configured!")
logger.info(
'Logging in with Brown Authentication as {}...'.format(username))
time.sleep(2)
# authenticate
username_field = driver.find_element_by_xpath(
"//input[@id='username']")
username_field.clear()
username_field.send_keys(username)
password_field = driver.find_element_by_xpath(
"//input[@id='password']")
password_field.clear()
password_field.send_keys(password)
password_field.send_keys(Keys.RETURN)
logger.info('Successfully logged into Brown as {}'.format(username))
time.sleep(2) # wait a few seconds for Duo to load
# Authenticate with DUO
cls.__duo_authenticator(driver, duo_bypass)
return
def click_sur_fiche_departement_nomgc(page: webdriver, Nomgc, Niveau):
elems = page.find_elements_by_xpath("//a[@href]")
for elem in elems:
print(Niveau, "elem:", elem.text, elem.get_attribute("href"))
if Nomgc in elem.text:
elem.click()
click_sur_fiche_departement_annee(elem, Niveau)
break
page.find_element_by_xpath(fiche_departement).click()
def buy(driver: webdriver):
url = 'https://market.m.taobao.com/app/sj/shop-membership-center/pages/index?spm=a1z10.4-b-s.w5003-22059585455.1.7f86274aL8QKmA' \
'&wh_weex=true&wx_navbar_transparent=true&sellerId=2360209412&scene=taobao_shop'
btn_buy = "//span[contains(text(),'1积分享兰蔻菁纯宝石唇膏 02')]/../following-sibling::div[1]//div[@class='gift-act-btn']"
btn_word = "//span[contains(text(),'1积分享兰蔻菁纯宝石唇膏 02')]/../following-sibling::div[1]//span[@class='gift-act-btn-text']"
btn = "//div[@class='btnWarp']"
word = "//span[@class='btn']"
count = 0
login_time = "2020-06-10 13:23:00"
buy_time = "2020-06-10 13:23:55"
isFirst = True
wait = WebDriverWait(driver, 10, 0.5)
while True:
if datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f') < buy_time:
# 提前登录
if isFirst and datetime.datetime.now().strftime(
'%Y-%m-%d %H:%M:%S.%f') >= login_time:
driver.maximize_window()
driver.get(url)
driver.implicitly_wait(2)
login(driver)
isFirst = False
continue
else:
# 如果连续抢60次都抢不到 那么就放弃吧
count += 1
if count > 60:
break
print("即将开始抢购...")
print(datetime.datetime.now())
# 重新进页面
driver.refresh()
# 开始页面的唇膏抢购按钮
buy_btn = wait.until(
lambda driver: driver.find_element_by_xpath(btn_buy))
# 开始页面的唇膏抢购按钮上的文字
word_btn = wait.until(
lambda driver: driver.find_element_by_xpath(btn_word))
if word_btn.text == '立即兑换':
# 如果抢购按钮上的文字为立即兑换 则点击按钮跳转详情页
buy_btn.click()
# 详情页中的购买按钮
targetBtn = wait.until(
lambda driver: driver.find_element_by_xpath(btn))
# 详情页中的购买按钮上的文字
targetBtnWord = wait.until(
lambda driver: driver.find_element_by_xpath(word))
if targetBtnWord.text != "已抢光":
# 如果详情页中购买按钮上的文字不为已枪光 则点击购买
targetBtn.click()
time.sleep(5)
# 抢到了就退出...
break
print("抢购结束...")
def close_popup(brow: webdriver) -> None:
accetto = './/button[@class="sc-ifAKCX ljEJIv"]'
try:
wait_clickable(brow, cfg.WAIT, accetto)
brow.find_element_by_xpath(accetto).click()
time.sleep(5)
except TimeoutException:
pass
def get_review_type(driver: webdriver, rid: str):
base_xp = Config.xp_review_misc_info_by_id(rid)
try:
driver.find_element_by_xpath(base_xp + '/span[1]/a/span')
return 'Verified Purchase'
except expected_conditions.NoSuchElementException as e:
try:
driver.find_element_by_xpath(base_xp + '/span[1]')
return 'Vine Customer Review of Free Product'
except expected_conditions.NoSuchElementException as e:
return 'Unverified Purchase'
def get_info(url, driver: webdriver, writer2, writer):
logging.error("开始爬数据，url：{}".format(url))
driver.get(url)
time.sleep(0.5)
# 职位名称
job_title = driver.find_element_by_xpath(
'//*[@id="root"]/div[3]/div/div/h3').text
# 工资
salary = driver.find_element_by_xpath(
'//*[@id="root"]/div[3]/div/div/div[2]/div[1]/span').text
# 职位关键字 list
try:
keyword = driver.find_element_by_xpath(
'//div[@class="highlights"]/div').text
except Exception as e:
keyword = "无"
# keyword = keyword.split("\n")
# 地点 省份
address = driver.find_element_by_xpath(
'//*[@id="root"]/div[3]/div/div/div[2]/div[1]/ul/li[1]/a').text
# 工作经验要求
work_experience = driver.find_element_by_xpath(
'//*[@id="root"]/div[3]/div/div/div[2]/div[1]/ul/li[2]').text
# 学历要求
education = driver.find_element_by_xpath(
'//*[@id="root"]/div[3]/div/div/div[2]/div[1]/ul/li[3]').text
# 公司名
company = driver.find_element_by_xpath('//div[@class="company"]/a[1]').text
# 要求
requirements = driver.find_element_by_xpath(
"//div[@class='describtion']").text
# 福利
welfare = driver.find_element_by_xpath(
'//*[@id="root"]/div[4]/div[1]/div[1]/div[1]/div').text
logging.error("开始写入文件")
# 写入文件
writer2.writerows([[
job_title.strip(),
salary.strip(),
address.strip(),
keyword.strip(),
work_experience.strip(),
education.strip(),
company.strip(),
welfare.strip(),
requirements.strip(), url
]])
writer.writerows([[
job_title.strip(),
salary.strip(),
address.strip(),
keyword.strip(),
work_experience.strip(),
education.strip(),
company.strip(),
welfare.strip(),
requirements.strip(), url
]])
logging.error("写入结束")
def login(driver: webdriver, username: str, password: str):
# Load page
driver.get("https://www.instagram.com/accounts/login/")
# Login
driver.find_element_by_xpath("//input[@name='username']").send_keys(username)
driver.find_element_by_xpath("//input[@name='password']").send_keys(password)
driver.find_element_by_xpath("//button[@type='submit']").click()
# Wait for 2FA or Profile link to appear
WebDriverWait(driver, 30).until(
EC.presence_of_element_located((By.LINK_TEXT, "Profile"))
)
def go_to_history(driver: webdriver) -> None:
"""
Navigates to the history page
Args:
driver (webdriver): The webdriver used to run the browser session
"""
# Go to History tab
driver.find_element_by_xpath('//*[@id="tabContainer"]/li[3]').click()
# Manually switch to History iframe after clicking the tab
driver.switch_to_frame('tab4frame')
ping_list = WebDriverWait(driver, 10).until(
EC.visibility_of_element_located(
(By.ID, 'ctl00_cphcontent_htPingList')))
ping_list.click()
def export(browser: webdriver):
browser.find_element_by_xpath('//els-btn[@text="Export"]/button').click()
sleep(3)
# select all columns
browser.find_element_by_xpath("//a[@ng-click='setSelectedAllColumns(true)']").click()
while 1:
try:
exportEle = WebDriverWait(browser, 60). \
until(EC.element_to_be_clickable((By.XPATH, '//a[.="Export as comma delimited (.csv)"]')))
# export csv
exportEle.click()
break
except Exception as e:
continue
def removeKataomoi(browser: webdriver, safe_accounts):
global remain_remove_count
"""
片思いったーにログインする
:param browser: webdriver
"""
# 片思いったーにアクセス
browser.get('http://kataomoi.net/redirect.php')
sleep(1)
url = browser.current_url
is_confirm = url.startswith(
"https://api.twitter.com/oauth/authorize?oauth_token")
if is_confirm:
submit_btn = browser.find_element_by_id("allow")
submit_btn.click()
sleep(1)
else:
# ログイン情報の入力
username_or_email = browser.find_element_by_xpath(
"//*[@id='username_or_email']")
username_or_email.send_keys(USER_NAME)
password = browser.find_element_by_xpath("//*[@id='password']")
password.send_keys(PASSWORD)
# ログイン
password.submit()
sleep(1)
browser.get('http://kataomoi.net/find_one_way.php')
sleep(1)
trs = browser.find_elements_by_tag_name("tr")
# 古い順位に並び替え
reversed_trs = reversed(trs)
print('▼フォロー解除中…▼')
for tr in reversed_trs:
if remain_remove_count <= 0:
break
tds = tr.find_elements_by_tag_name("td")
if len(tds) > 1 and not (tds[1].find_element_by_tag_name("a").text
in safe_accounts):
print(tds[1].find_element_by_tag_name("a").text)
tr.find_elements_by_tag_name("span")[0].click()
remain_remove_count = remain_remove_count - 1
sleep(0.5)
def process_action(self, driver: webdriver, key: str, action: str) -> None:
if action == "xpath":
driver.find_element_by_xpath(key).click()
if action == "click_id":
driver.find_element_by_id(key).click()
if action == "location":
driver.find_element_by_id(key).send_keys(self.requirements[action])
driver.find_element_by_id(key).send_keys(Keys.TAB)
if action in ["beds_min", "beds_max"]:
select = Select(driver.find_element_by_xpath(key))
select.select_by_value(self.requirements[action])
if action == "link_text":
driver.find_element_by_link_text(key).click()
if action == "css":
driver.find_element_by_css_selector(key).click()
def get_player_name(player_elem: webdriver) -> str:
name = player_elem.find_element_by_xpath(
'.//span[@class="player-name ellipsis"]')
name = name.get_attribute('innerText')
return format_player_name(player_name=name)
def open_profile_options(brow: webdriver) -> None:
path = './/a[@title="Profilo"]'
wait_clickable(brow, path)
button = brow.find_element_by_xpath(path)
button.click()
time.sleep(5)
def open_profile_history(brow: webdriver) -> None:
path = './/a[@title="Movimenti e giocate"]'
wait_clickable(brow, path)
button = brow.find_element_by_xpath(path)
button.click()
time.sleep(5)
def return_to_league_page(brow: webdriver) -> None:
back_path = './/a[@class="back-competition ng-scope"]'
back = brow.find_element_by_xpath(back_path)
scroll_to_element(brow, back)
wait_clickable(brow, back_path)
back.click()
def extract_bet_quote(bet_element: webdriver) -> float:
quote_element = bet_element.find_element_by_xpath(
'.//div[@class="selection-price"]')
quote = quote_element.get_attribute('innerText').upper()
return float(quote)
def get_review_text(driver: webdriver,
n_reviews: int = 100,
verbose=True) -> str:
'''Given a webdriver object, will collect all loaded reviews on page'''
res = []
start = 5
temp = '/html/body/div[3]/div[5]/main/div[2]/div[2]/div/div[1]/div/div[%s]/div[1]/div[2]/div[1]'
for i in range(n_reviews):
path = temp % str(start)
try:
text = driver.find_element_by_xpath(path)
if verbose:
print(f'Review #{i} Success')
except:
text = f'Review #{i} Error'
if verbose:
print(text)
try:
res.append(text.text)
except:
res.append(text)
start += 2
time.sleep(0.2)
return res
def get_cattle_prices_for_a_date(driver: webdriver, date_to_pick: date):
# wait for the first date picker item to appear, click it and send the date
wait = WebDriverWait(driver, 10)
first_date_picker = wait.until(
ec.visibility_of_element_located(
(By.XPATH, '//*[@id="datepicker1"]')))
first_date_picker.click()
first_date_picker.clear()
first_date_picker.send_keys(date_to_pick.strftime('%d/%m/%Y'))
# wait for the second date picker item to appear, click it and send the date
wait = WebDriverWait(driver, 10)
second_date_picker = wait.until(
ec.visibility_of_element_located(
(By.XPATH, '//*[@id="datepicker2"]')))
second_date_picker.click()
second_date_picker.clear()
second_date_picker.send_keys(date_to_pick.strftime('%d/%m/%Y'))
# find the execute query button and press enter
# Find the query button and press enter
query_button = driver.find_element_by_xpath('//*[@id="Aceptar"]')
query_button.send_keys(Keys.ENTER)
page_source = driver.page_source
return page_source
def launch_twitter(driver: webdriver):
"""
:type driver: selenium.webdriver.firefox.webdriver.WebDriver
"""
twitter_url = "https://twitter.com/search?f=tweets&vertical=default&q=gleam.io&src=typd&lang=en"
driver.get(twitter_url)
URLs = driver.find_elements_by_css_selector("li[data-item-id]")
lurl = []
for tweet in URLs:
if tweet.find_elements_by_class_name("twitter-timeline-link"):
linkr = tweet.find_element_by_class_name("twitter-timeline-link")
text = linkr.get_attribute("href")
if len(tweet.find_elements_by_class_name("card2")) == 0 and len(
text) == 0:
if len(
tweet.find_elements_by_xpath(
".//*[starts-with(@id,'xdm')]")) != 0:
frame = tweet.find_element_by_xpath(
".//*[starts-with(@id,'xdm')]")
driver.switch_to.frame(frame)
link = driver.find_element_by_xpath("/html/body/div/div/a")
text = link.get_attribute("href")
driver.switch_to.default_content()
lurl.append(text)
return lurl
def identify_groupement_commune(page: webdriver) -> (str, str):
"""
:param page:
:return: Nom et reference du groupement de commune
"""
# Identification du groupement de commune
nomcc = page.find_element_by_xpath('// *[@id="gfp"]').text
nomccs = str(nomcc).replace("/", "_")
# print("identify _groupement_commune-nomccs", nomccs, "nomcc", nomcc)
# Si la cc à déjà été vue
if nomcc in listecc:
# Renvoie la référence
idx = listecc.index(nomcc)
print("cc deja dans la liste'", nomccs, refcc[idx], refccnom[idx],
listecc)
return nomccs, refcc[idx], refccnom[idx]
else:
# Ajoute la cc dans la liste 'déjà vue' et envoie la référence
idxcc = len(listecc)
listecc.append(nomccs)
# print("liste augmentee",listecc)
refcc.append('*'.join((nodep, str(idxcc).zfill(3))))
refccnom.append('*'.join((nomccs, nodep, str(idxcc).zfill(3))))
print("liste augmentée", nomccs, refcc[-1], refccnom[-1], listecc)
return nomccs, refcc[-1], refccnom[-1]
def get_property(element: webdriver, prop_xpath: str) -> str:
prop = ""
try:
prop = element.find_element_by_xpath(f".{prop_xpath}").text
except:
pass
return prop
def place_bet(brow: webdriver) -> None:
button_location = './/div[@class="buttons-betslip"]'
button = brow.find_element_by_xpath(button_location)
scroll_to_element(brow, button)
time.sleep(5)
button.click()
time.sleep(10)
def refresh_money(brow: webdriver) -> None:
refresh_path = './/user-balance-refresh-btn'
wait_clickable(brow, refresh_path)
refresh = brow.find_element_by_xpath(refresh_path)
scroll_to_element(brow, refresh)
refresh.click()
def extract_bet_name(bet_element: webdriver) -> str:
name_element = bet_element.find_element_by_xpath(
'.//div[@class="selection-name ng-binding"]')
name = name_element.get_attribute('innerText').upper()
if '(' in name:
name = name.split('(')[0]
return name.strip()
def review_click_six(driver: webdriver, n_reviews: int = 100) -> None:
'''Given a webdriver object, will click the "6 more reviews" button in order to load page'''
start = 16
temp = '/html/body/div[3]/div[5]/main/div[2]/div[2]/div/div[1]/div/div[%s]/button'
for x in range(n_reviews // 6):
path = temp % str(start)
print(f'XPath Attempt at {path[:-15]}\nFill: {start}')
success = False
while not success:
try:
driver.find_element_by_xpath(path).click()
time.sleep(3)
start += 12
success = True
except:
time.sleep(1)
