def execSearch(browser: webdriver, inifile):
# 検索ワード抽出
browser.get(_get(inifile, 'search', 'url_trend'))
contents = browser.find_elements_by_tag_name('h1')
searchWords = []
for content in contents:
searchWords.append(content.text)
searchWords.pop(0)
searchWords.pop(0)
# ログインボタンの押下(検索、メールDEポイント共通)
browser.get(_get(inifile, 'search', 'url') + '/Web?qt=' + searchWords[0])
searchWords.pop(0)
browser.find_element_by_link_text(_get(inifile, 'search',
'login_context')).click()
login_user = browser.find_element_by_name(_get(inifile, 'user', 'id_name'))
login_user.send_keys(_get(inifile, 'user', 'id'))
login_password = browser.find_element_by_name(
_get(inifile, 'user', 'pass_name'))
login_password.send_keys(_get(inifile, 'user', 'pass'))
browser.find_element_by_name("submit").click()
sleep(_getRandomNum())
## 検索ワードを一つずつ処理
for index, searchWord in enumerate(searchWords):
search_box = browser.find_element_by_name("qt")
search_box.clear()
search_box.send_keys(searchWord)
browser.find_element_by_id('searchBtn').click()
sleep(_getRandomNum())
def go_to_postings(web_driver: webdriver, site: str) -> None:
if site == OSCAR:
web_driver.get(oscar_postings_url)
web_driver.find_element_by_link_text(
"View all available postings").click()
if site == WORKS:
web_driver.get(water_postings_url)
web_driver.find_element_by_css_selector(
".btn btn-primary btn-small".replace(' ', '.')).click()
def execLogin(browser: webdriver, inifile):
# Googleにアクセス
browser.get(_get(inifile, 'init', 'url'))
# ログインボタンの押下
browser.find_element_by_link_text(_get(inifile, 'init', 'login_context')).click()
sleep(_getRandomNum())
login_user = browser.find_element_by_name(_get(inifile, 'user', 'id_name'))
login_user.send_keys(_get(inifile, 'user', 'id'))
login_password = browser.find_element_by_name(_get(inifile, 'user', 'pass_name'))
login_password.send_keys(_get(inifile, 'user', 'pass'))
browser.find_element_by_name("submit").click()
sleep(_getRandomNum())
def process_action(self, driver: webdriver, key: str, action: str) -> None:
if action == "xpath":
driver.find_element_by_xpath(key).click()
if action == "click_id":
driver.find_element_by_id(key).click()
if action == "location":
driver.find_element_by_id(key).send_keys(self.requirements[action])
driver.find_element_by_id(key).send_keys(Keys.TAB)
if action in ["beds_min", "beds_max"]:
select = Select(driver.find_element_by_xpath(key))
select.select_by_value(self.requirements[action])
if action == "link_text":
driver.find_element_by_link_text(key).click()
if action == "css":
driver.find_element_by_css_selector(key).click()
def work_details_request(browser: webdriver, title: str) -> BeautifulSoup:
wait()
try:
work = browser.find_element_by_link_text(title)
# Waiting 'till link is clickable
while True:
try:
work.click()
break
except ElementClickInterceptedException:
continue
requestmeter.count()
sleep(0.5) # This delay permits the html display entirely
html_ = browser.page_source
close_button = browser.find_element_by_id('gs_md_cita-d-x')
close_button.click()
except Exception as err:
print("!!!>>>", title)
print(err)
logging_collector(
"ERROR",
"NOT CLICKABLE LNK",
[
browser.find_element_by_id('gsc_prf_in').text, # Author
title, # Title
err
]) # Error
return None
return BeautifulSoup(html_, 'html.parser')
def get_restaurant_links_from_all_pages(driver: webdriver, start_url: str):
print(f"Started at {datetime.now().strftime('%H:%M:%S')}")
r_links = []
url = start_url
page_num = 1
while page_num <= 408:
soup = get_soup_using_selenium(url, driver, 15)
try:
r_links = get_restaurant_links_from_a_page(soup)
write_restaurant_link_to_file(
r_links, THIS_DIR / 'data/tripadvisor/restaurant_links_t.txt')
print(f"Finished page {page_num}")
page_num += 1
url = driver.find_element_by_link_text('Next').get_attribute(
'href')
if page_num >= 80:
sleep(30)
elif page_num >= 40:
sleep(15)
else:
sleep(10)
except AttributeError:
print(r_links)
print(f"Stopped at {datetime.now().strftime('%H:%M:%S')}")
raise Exception(
f"Is suspected to be a robot at page {page_num}. Try again.")
print(
f"Finished at {datetime.now().strftime('%H:%M:%S')} (page {page_num})")
return r_links
def navigate_to_archive(driver: webdriver):
find_2fa_ready_element(driver).click()
# Select GESAMTER_ZEITRAUM
zeitraum_select = driver.find_element_by_id('f1-zeitraumInput_pbInput')
select_field = webdriver.support.ui.Select(zeitraum_select)
select_field.select_by_value('GESAMTER_ZEITRAUM')
# Start search
suchen_field = driver.find_element_by_link_text('Suchen')
suchen_field.click()
def bunkyo_court_scraper(court_number, browser: webdriver, mysql):
# スクリーンショットのファイル名用に日付を取得
dt = datetime.datetime.today()
dtstr = dt.strftime("%Y%m%d%H%M%S")
browser.get("https://www.yoyaku.city.bunkyo.lg.jp/reselve/m_index.do")
sleep(2)
browser.find_element_by_link_text("施設空き状況").click()
sleep(2)
browser.find_element_by_link_text("利用目的から選ぶ").click()
sleep(2)
browser.find_element_by_link_text("利用目的名から選択").click()
sleep(2)
browser.find_element_by_css_selector("input[type='submit']").click()
sleep(2)
browser.find_element_by_css_selector("input[type='submit']").click()
sleep(2)
browser.find_element_by_link_text("竹早テニスコート（土日祝）").click()
sleep(2)
court_number_full = jaconv.h2z(str(court_number), digit=True)
browser.find_element_by_link_text(f"第{court_number_full}コート土日祝").click()
sleep(2)
element = browser.find_element_by_css_selector("form")
schedule_parser(element.text, court_number, mysql)
for i in range(8):
browser.find_element_by_css_selector("input[value=' 次の週 ']").click()
sleep(2)
element = browser.find_element_by_css_selector("form")
schedule_parser(element.text, court_number, mysql)
def access_group(driver: webdriver, name: str):
"""
Access the group with the webdriver.
:param driver: Selenium webdriver instance that is logged into facebook.
:param name: Name of the group to access.
"""
groups = driver.find_element_by_link_text("Groups")
groups.click()
Wait(driver).until(EC.presence_of_element_located(
(By.LINK_TEXT, name))).click()
Wait(driver).until(
EC.presence_of_element_located((By.ID, "newsFeedHeading")))
def into_live(driver: webdriver):
"""
进入直播页面,标签页面可变更
"""
driver.find_element_by_link_text(u"直播").click()
def go_to_problem_page(driver: webdriver, problem_name: str):
driver.get(f'{WEBSITE_ROOT}/problemset')
driver.find_element_by_link_text(problem_name).click()
print('go_to_problem_page:', driver.current_url)
def find_elm_by_link_text(driver: webdriver, text: str):
return driver.find_element_by_link_text(text)
def find_2fa_ready_element(driver: webdriver):
return driver.find_element_by_link_text('Archiv')
def CLR_Html(browser: webdriver, Name: str) -> str:
"""
a = CLR_Html(self.b, ['page_name', 'profile_message_send', 'profile_action_btn', 'profile_msg_split'])
print(a)
:param Name:
:param browser:
:return:
"""
res = []
all_res = []
# a = Soport_webdriver.CLR_Html(b,['mail_box_send'])
for name in Name:
try:
all_res.append(browser.find_element_by_id(name).text)
res.append('find_element_by_id')
except:
pass
try:
all_res.append(browser.find_element_by_name(name).text)
res.append('find_element_by_name')
except:
pass
try:
all_res.append(browser.find_element_by_xpath(name).text)
res.append('find_element_by_xpath')
except:
pass
try:
all_res.append(browser.find_element_by_link_text(name).text)
res.append('find_element_by_link_text')
except:
pass
try:
all_res.append(
browser.find_element_by_partial_link_text(name).text)
res.append('find_element_by_partial_link_text')
except:
pass
try:
all_res.append(browser.find_element_by_tag_name(name).text)
res.append('find_element_by_tag_name')
except:
pass
try:
all_res.append(browser.find_element_by_class_name(name).text)
res.append('find_element_by_class_name')
except:
pass
try:
all_res.append(browser.find_element_by_css_selector(name).text)
res.append('find_element_by_css_selector')
except:
pass
io = ''
for x in range(len(res)):
io += '{} |-| {} |-| {}\n'.format(str(Name[x]), str(res[x]),
str(all_res[x]))
return io
