def download_data(browser_object: webdriver,
config_object: configparser.ConfigParser) -> BeautifulSoup:
browser_object.get(config_object["rekyl_portal"]["url"])
time.sleep(1)
username = browser_object.find_element_by_id("username")
password = browser_object.find_element_by_css_selector(
"input[type=password")
username.send_keys(config_object["rekyl_portal"]["username"])
password.send_keys(config_object["rekyl_portal"]["password"])
browser_object.find_element_by_id("button_login_security_low").click()
time.sleep(3)
show_errands = Select(browser_object.find_element_by_name("maxhits"))
show_errands.select_by_index(10)
time.sleep(5)
browser_object.find_element_by_name("maxhits").send_keys(Keys.RETURN)
time.sleep(5)
iframe = browser_object.find_element_by_id("iframe_workorder")
browser_object.switch_to.frame(iframe)
iframe_source = browser_object.page_source
soup = BeautifulSoup(iframe_source, "html.parser")
browser_object.close()
browser_object.quit()
logging.info("Raw data downloaded")
return soup
def extract_info_from_iframe(browser: webdriver) -> []:
try:
# open all other pages with comments
button = browser.find_element_by_css_selector(
"button[data-dot='strankovani/nacist_dalsi']")
while button:
try:
action = ActionChains(browser)
action.move_to_element(button).click().perform()
except (ElementClickInterceptedException,
StaleElementReferenceException):
pass
sleep(0.5 * SETTINGS["lazy_factor"])
button = browser.find_element_by_css_selector(
"button[data-dot='strankovani/nacist_dalsi']")
except NoSuchElementException:
pass
try:
# open threads of subcomments
for button in browser.find_elements_by_css_selector(
"button[data-dot='nacist_nove_podkomentare']"):
try:
action = ActionChains(browser)
action.move_to_element(button).click().perform()
except (ElementClickInterceptedException,
StaleElementReferenceException):
pass
sleep(0.3 * SETTINGS["lazy_factor"])
except NoSuchElementException:
pass
# now we can use familiar beautiful soup
soup = BeautifulSoup(browser.page_source, "html.parser")
authors = soup.select("a[class='f_bO'] span")
texts = soup.select("p[class='d_aJ']")
reactions = soup.select("a[class='f_cQ']")
comments = []
progress_bar = tqdm(total=len(authors), desc="Comments", position=0)
for i in range(len(authors)):
author = authors[i].text
text = texts[i].text
# expect that reaction can be missing in some comments
reactions_count = 0
if len(reactions) > i:
reactions_count = reactions[i].text
comments.append(
Comment(author=author, text=text, reactions=reactions_count))
progress_bar.update(1)
# browser doesn't need to be open from now on
browser.quit()
return comments
def setup_proxy_for_driver(driver: webdriver, test_url=None, times=0):
if times > 9:
logger.warning('setup_proxy_for_driver no available proxy')
raise TooManyTimesException('setup_proxy_for_driver')
try:
try:
# clean cookie and close session
driver.delete_all_cookies()
driver.quit()
except:
pass
proxy_url = get_proxy().get('proxy')
logger.info('proxy get {}'.format(proxy_url))
capabilities = get_capabilities(proxy_url)
logger.info('start new session')
driver.start_session(capabilities=capabilities)
logger.info('start testing proxy')
ok = test_proxy(driver, test_url, proxy_url)
if not ok:
logger.warning('proxy checking failed for {} times'.format(times +
1))
return setup_proxy_for_driver(driver, test_url, times=times + 1)
logger.info('proxy works')
return driver
except SessionNotCreatedException:
logger.error('Failed to start a new session')
return setup_proxy_for_driver(connect_to_driver(),
test_url,
times=times + 1)
except InvalidSessionIdException as e2:
logger.error('Session id invalid {}'.format(e2))
return setup_proxy_for_driver(driver, test_url, times=times + 1)
except WebDriverException as e3:
logger.error('No active session with ID')
return setup_proxy_for_driver(driver, test_url, times=times + 1)
except NoProxyAvailableException:
logger.error('No proxy')
mongo._report_error({
'error_source': 'proxy',
'message': 'no_proxy_available',
'url': None,
'payload': {}
})
return setup_proxy_for_driver(driver, test_url, times=times + 1)
except Exception as e:
logger.error(f'setup_proxy_for_driver {e}')
raise e
def navigateLogin(driver: webdriver) -> HtmlChecker:
driver.save_screenshot("Post_Login.png")
resultBtn = driver.find_element_by_xpath("//a[@data-id='21']")
resultBtn.click()
time.sleep(10)
driver.save_screenshot("Results.png")
pageSource = driver.page_source
HtmlObj = HtmlChecker(pageSource)
driver.quit()
return HtmlObj
def track_time(session: webdriver) -> None:
try:
st = session.find_element_by_id('div-fichaje-action')
print(st.text)
st.click()
print('waiting to save track info')
sleep(10)
print("Time tracked")
except Exception as e:
print(f'Unable to submit track info:{e}')
finally:
session.quit()
def close_firefox(driver: webdriver,
restart: bool = False,
lastp: bool = False,
no_ads_found: bool = False,
time_exc: bool = False) -> None:
"""Close, quit, destroy the webdriver instance of Firefox and safely close the DB connection."""
if restart:
print('Restart of Firefox requested.\n' 'Closing Firefox...\n')
driver.close()
driver.quit()
elif time_exc:
print('Failed to load the listing URL in time.\n'
'Closing Firefox and retrying...\n')
driver.close()
driver.quit()
elif lastp:
print(
'This was the last page that was set or found in the first listing URL.\n'
'Closing Firefox...\n')
driver.close()
driver.quit()
disconnect_db()
elif no_ads_found:
print('No ads found on the page. Breaking out of the loop.\n'
'Closing Firefox...\n')
driver.close()
driver.quit()
disconnect_db()
def logout(driver: webdriver, manager: str) -> None:
'''
This function log you out and gracefully quits everything.
:param driver: The web browser driver.
:param manager: The Address of the Manager/FMC.
:return: None
'''
time.sleep(5)
if manager.startswith('http'):
driver.get('{}/login.cgi?logout=1'.format(manager))
else:
driver.get('https://{}/login.cgi?logout=1'.format(manager))
time.sleep(5)
driver.quit() # Gracefully quits everything.
def _selenium_operations(driver: webdriver, user_name: str,
user_password: str):
# 登录界面
Util.info_print('登录界面', 3)
driver.get('https://users.nexusmods.com/auth/sign_in')
Util.info_print('请在页面中登录N网账户', 3)
Util.info_print('如果设置在conf.ini的账户密码正确，这个过程会自动完成。', 3)
Util.info_print('如果不正确，请手动输入账户密码', 3)
Util.info_print('每一步操作都设置了一定的的可行时间，超过时间程序就会退出', 3)
wait = WebDriverWait(driver, 300)
username_inputer = wait.until(
presence_of_element_located((By.ID, "user_login")))
userpassword_inputer = wait.until(
presence_of_element_located((By.ID, "password")))
commit_button = wait.until(
presence_of_element_located((By.XPATH, '//input[@type="submit"]')))
username_inputer.send_keys(user_name)
userpassword_inputer.send_keys(user_password)
commit_button.click()
wait.until(EC.url_changes)
# 欢迎界面
index_a = wait.until(
EC.presence_of_element_located(
(By.XPATH, '//div[@class="links"]/div[@class="left-link"]/a[1]')))
index_a.click()
Util.info_print('等待进入首页，请勿操作', 3)
# 返回首页后
# Util.info_print('等待从首页中获取cookies', 3)
wait.until(
EC.presence_of_element_located(
(By.CSS_SELECTOR, ".intro > h1:nth-child(1)")))
nexus_cookies_list = driver.get_cookies()
driver.quit()
nexus_cookies = dict()
for cookie in nexus_cookies_list:
nexus_cookies[cookie['name']] = cookie['value']
return nexus_cookies
def _selenium_operations(driver: webdriver, user_name: str,
user_password: str):
# 登录界面
_info_print('Coo_5')
driver.get('https://users.nexusmods.com/auth/sign_in')
_info_print('Coo_6')
_info_print('Coo_7')
_info_print('Coo_8')
_info_print('Coo_9')
wait = WebDriverWait(driver, 300)
username_inputer = wait.until(
presence_of_element_located((By.ID, "user_login")))
userpassword_inputer = wait.until(
presence_of_element_located((By.ID, "password")))
commit_button = wait.until(
presence_of_element_located((By.XPATH, '//input[@type="submit"]')))
username_inputer.send_keys(user_name)
userpassword_inputer.send_keys(user_password)
commit_button.click()
wait.until(EC.url_changes)
# 欢迎界面
index_a = wait.until(
EC.presence_of_element_located(
(By.XPATH, '//div[@class="links"]/div[@class="left-link"]/a[1]')))
index_a.click()
_info_print('Coo_10')
# 返回首页后
# Util.info_print('等待从首页中获取cookies', 3)
wait.until(
EC.presence_of_element_located(
(By.CSS_SELECTOR, ".intro > h1:nth-child(1)")))
nexus_cookies_list = driver.get_cookies()
driver.quit()
nexus_cookies = dict()
for cookie in nexus_cookies_list:
nexus_cookies[cookie['name']] = cookie['value']
return nexus_cookies
def CaptureStats(self, driver: webdriver) -> HubStatsRecord:
"""
Capture stats from a BT Smart Hub 2 and return a populated HubStatsRecord
:param driver: A configured Selenium webdriver object
"""
print("Capturing modem stats from BT Smart Hub 2 at " + self.URI +
"\n")
# Capture the data from the Smart Hub
# Start the web session
driver.get(self.URI)
# Select "Advanced Settings" screen
WebDriverWait(driver, 30).until(
expected_conditions.element_to_be_clickable(
(By.ID, 'gotoA'))).click()
# Select "Helpdesk"
WebDriverWait(driver, 30).until(
expected_conditions.element_to_be_clickable(
(By.ID, 'advHelpdesk'))).click()
#Put the password int he dialog"
WebDriverWait(driver, 30).until(
expected_conditions.visibility_of_element_located(
(By.ID,
'login_password_input_noshow'))).send_keys(self.AdminPassword)
#Click the "OK" buttin
WebDriverWait(driver, 30).until(
expected_conditions.element_to_be_clickable(
(By.ID, 'ok_button'))).click()
#Grab the infobox
DataTableWebElement = WebDriverWait(driver, 10).until(
expected_conditions.element_to_be_clickable((By.ID, 'Infobox')))
# Process uptime
UptimeStrings = re.findall(
r"[-+]?\d*\.\d+|\d+",
DataTableWebElement.find_element_by_id('DSLuptime').text)
Uptime = int(UptimeStrings[0]) * 24 + int(UptimeStrings[1]) * 1 + int(
UptimeStrings[2]) / 60 + int(UptimeStrings[3]) / 3600
# Process Data rate
DatarateStrings = re.findall(
r"[-+]?\d*\.\d+|\d+",
DataTableWebElement.find_element_by_id('Datarate').text)
DatarateValues = [float(i) for i in DatarateStrings]
# Process Max data rate
MDatarateStrings = re.findall(
r"[-+]?\d*\.\d+|\d+",
DataTableWebElement.find_element_by_id('MDatarate').text)
MDatarateValues = [float(i) for i in MDatarateStrings]
# Process noise margin
NoiseMarginStrings = re.findall(
r"[-+]?\d*\.\d+|\d+",
DataTableWebElement.find_element_by_id('noise_margin').text)
NoiseMarginValues = [float(i) for i in NoiseMarginStrings]
# Close Web browser
driver.quit()
# Populate the return data structure
d = HubStatsRecord()
d.Uptime = Uptime
d.DataRate_Upload = DatarateValues[0]
d.DataRate_Down = DatarateValues[1]
d.MaxDataRate_Up = MDatarateValues[0]
d.MaxDataRate_Down = MDatarateValues[1]
d.SNR_Up = NoiseMarginValues[0]
d.SNR_Down = NoiseMarginValues[1]
return d
def quit_chromium(driver: webdriver) -> None:
driver.quit()
if operating_system() != "Windows":
run(['osascript', '-e', 'quit app \"Chromium\"'])
def cita_selection(driver: webdriver, context: CustomerProfile):
resp_text = body_text(driver)
if "Por favor, valide el Captcha para poder continuar" in resp_text:
success = process_captcha(driver, context, partially=True)
if not success:
return None
return phone_mail(driver, context, retry=True)
elif "DISPONE DE 5 MINUTOS" in resp_text:
logging.info("Cita attempt -> selection hit! :)")
if context.save_artifacts:
driver.save_screenshot(f"citas-{datetime.datetime.now()}.png".replace(":", "-"))
try:
driver.find_elements_by_css_selector("input[type='radio'][name='rdbCita']")[
0
].send_keys(Keys.SPACE)
except Exception as e:
logging.error(e)
pass
driver.execute_script("envia();")
time.sleep(0.5)
driver.switch_to.alert.accept()
elif "Seleccione una de las siguientes citas disponibles" in resp_text:
logging.info("Cita attempt -> selection hit! :)")
if context.save_artifacts:
driver.save_screenshot(f"citas-{datetime.datetime.now()}.png".replace(":", "-"))
try:
slots = driver.find_elements_by_css_selector("#CitaMAP_HORAS tbody [id^=HUECO]")
slot_ids = sorted([*map(lambda x: x.get_attribute("id"), slots)])
if slot_ids:
slot = slot_ids[0]
driver.execute_script(f"confirmarHueco({{id: '{slot}'}}, {slot[5:]});")
driver.switch_to.alert.accept()
except Exception as e:
logging.error(e)
return None
else:
logging.info("Cita attempt -> missed selection :(")
return None
# 8. Confirmation
resp_text = body_text(driver)
if "Debe confirmar los datos de la cita asignada" in resp_text:
logging.info("Cita attempt -> confirmation hit! :)")
if context.telegram_token:
dispatcher = context.updater.dispatcher
def shutdown():
context.updater.stop()
context.updater.is_idle = False
def code_received(update, ctx):
logging.info(f"Received code: {ctx.args[0]}")
element = driver.find_element_by_id("txtCodigoVerificacion")
element.send_keys(ctx.args[0])
driver.find_element_by_id("chkTotal").send_keys(Keys.SPACE)
driver.find_element_by_id("enviarCorreo").send_keys(Keys.SPACE)
btn = driver.find_element_by_id("btnConfirmar")
btn.send_keys(Keys.ENTER)
resp_text = body_text(driver)
ctime = datetime.datetime.now()
if "CITA CONFIRMADA Y GRABADA" in resp_text:
context.bot_result = True
code = driver.find_element_by_id("justificanteFinal").text
logging.info(f"Justificante cita: {code}")
caption = f"Cita confirmed! {code}"
if context.save_artifacts:
image_name = f"CONFIRMED-CITA-{ctime}.png".replace(":", "-")
driver.save_screenshot(image_name)
ctx.bot.send_photo(
chat_id=update.effective_chat.id,
photo=open(os.path.join(os.getcwd(), image_name), "rb"),
caption=caption,
)
btn = driver.find_element_by_id("btnImprimir")
btn.send_keys(Keys.ENTER)
else:
ctx.bot.send_message(chat_id=update.effective_chat.id, text=caption)
threading.Thread(target=shutdown).start()
return True
elif "Lo sentimos, el código introducido no es correcto" in resp_text:
ctx.bot.send_message(
chat_id=update.effective_chat.id, text="Incorrect, please try again"
)
else:
error_name = f"error-{ctime}.png".replace(":", "-")
driver.save_screenshot(error_name)
ctx.bot.send_photo(
chat_id=update.effective_chat.id,
photo=open(os.path.join(os.getcwd(), error_name), "rb"),
)
ctx.bot.send_message(
chat_id=update.effective_chat.id, text="Something went wrong"
)
dispatcher.add_handler(CommandHandler("code", code_received, pass_args=True))
context.updater.start_polling(poll_interval=1.0)
for i in range(5):
speaker.say("ALARM")
# Waiting for response 5 minutes
time.sleep(360)
threading.Thread(target=shutdown).start()
if context.save_artifacts:
driver.save_screenshot(
f"FINAL-SCREEN-{datetime.datetime.now()}.png".replace(":", "-")
)
if context.bot_result:
driver.quit()
os._exit(0)
return None
else:
for i in range(10):
speaker.say("ALARM")
logging.info("Press Any button to CLOSE browser")
input()
driver.quit()
os._exit(0)
else:
logging.info("Cita attempt -> missed confirmation :(")
if context.save_artifacts:
driver.save_screenshot(
f"failed-confirmation-{datetime.datetime.now()}.png".replace(":", "-")
)
return None
def fetch_image_urls(search_engine: SearchEngine, query: str,
max_img_to_fetch: int, driver: webdriver):
try:
driver.get(search_engine.url)
search_box = driver.find_element_by_css_selector(
search_engine.css_selector_search_line)
search_box.send_keys(query)
search_box.submit()
except Exception as ex:
driver.quit()
print(f"Exception {ex} with search box.")
return None
image_urls = []
image_count = 0
results_start = 0
while image_count < max_img_to_fetch:
scroll_to_end(driver, time_sleep=random.randint(1, 3))
# get all image thumbnail results
thumbnail_results = driver.find_elements_by_css_selector(
search_engine.css_selector_thumbnail_img)
number_results = len(thumbnail_results)
print(
f"Found: {number_results} search results. Extracting links from {results_start}:{number_results}"
)
for img in thumbnail_results[results_start:number_results]:
# try to click every thumbnail such that we can get the real image behind it
try:
img.click()
time.sleep(random.randint(1, 3))
except Exception:
continue
# extract image urls
actual_images = driver.find_elements_by_css_selector(
search_engine.css_selector_image_url)
for actual_image in actual_images:
if actual_image.get_attribute(
'src') and 'http' in actual_image.get_attribute('src'):
image_urls.append(actual_image.get_attribute('src'))
break
try:
close_preview = driver.find_element_by_css_selector(
search_engine.close_preview)
close_preview.click()
except selenium_exceptions.ElementNotVisibleException as ex:
print(ex)
continue
except Exception as ex:
print(f'Error with close_preview: {ex}')
return image_urls
image_count = len(image_urls)
if len(image_urls) >= max_img_to_fetch:
print(f"Found: {len(image_urls)} image links, done!")
break
else:
print("Found:", len(image_urls),
"image links, looking for more ...")
time.sleep(random.randint(10, 30))
if search_engine is GOOGLE_IMG:
load_more_button = driver.find_element_by_css_selector(
".mye4qd")
if load_more_button:
driver.execute_script(
"document.querySelector('.mye4qd').click();")
# move the result startpoint further down
results_start = len(thumbnail_results)
return image_urls
def quit_session(driver: webdriver):
driver.quit()
def close_browser(driver: webdriver):
driver.close()
driver.quit()
def cita_selection(driver: webdriver, context: CustomerProfile):
try:
WebDriverWait(driver, DELAY).until(
EC.presence_of_element_located((By.TAG_NAME, "body")))
except TimeoutException:
logging.error("Timed out waiting for body to load")
return None
resp_text = driver.find_element_by_tag_name("body").text
if "DISPONE DE 5 MINUTOS" in resp_text:
logging.info("Cita attempt -> selection hit! :)")
if context.save_artifacts:
driver.save_screenshot(
f"citas-{datetime.datetime.now()}.png".replace(":", "-"))
try:
driver.find_elements_by_css_selector(
"input[type='radio'][name='rdbCita']")[0].send_keys(Keys.SPACE)
except Exception as e:
logging.error(e)
pass
btn = driver.find_element_by_id("btnSiguiente")
btn.send_keys(Keys.ENTER)
driver.switch_to.alert.accept()
else:
logging.info("Cita attempt -> missed selection :(")
return None
# 9. Confirmation
try:
WebDriverWait(driver, DELAY).until(
EC.presence_of_element_located((By.TAG_NAME, "body")))
except TimeoutException:
logging.error("Timed out waiting for body to load")
return None
resp_text = driver.find_element_by_tag_name("body").text
if "Debe confirmar los datos de la cita asignada" in resp_text:
logging.info("Cita attempt -> confirmation hit! :)")
if context.telegram_token:
dispatcher = context.updater.dispatcher
def shutdown():
context.updater.stop()
context.updater.is_idle = False
def code_received(update, ctx):
logging.info(f"Received code: {ctx.args[0]}")
element = driver.find_element_by_id("txtCodigoVerificacion")
element.send_keys(ctx.args[0])
driver.find_element_by_id("chkTotal").send_keys(Keys.SPACE)
driver.find_element_by_id("enviarCorreo").send_keys(Keys.SPACE)
btn = driver.find_element_by_id("btnConfirmar")
btn.send_keys(Keys.ENTER)
try:
WebDriverWait(driver, DELAY).until(
EC.presence_of_element_located((By.TAG_NAME, "body")))
except TimeoutException:
logging.info("Timed out waiting for body to load")
return None
resp_text = driver.find_element_by_tag_name("body").text
ctime = datetime.datetime.now()
if "CITA CONFIRMADA Y GRABADA" in resp_text:
context.bot_result = True
code = driver.find_element_by_id("justificanteFinal").text
logging.info(f"Justificante cita: {code}")
caption = f"Cita confirmed! {code}"
if context.save_artifacts:
image_name = f"CONFIRMED-CITA-{ctime}.png".replace(
":", "-")
driver.save_screenshot(image_name)
ctx.bot.send_photo(
chat_id=update.effective_chat.id,
photo=open(os.path.join(os.getcwd(), image_name),
"rb"),
caption=caption,
)
btn = driver.find_element_by_id("btnImprimir")
btn.send_keys(Keys.ENTER)
else:
ctx.bot.send_message(chat_id=update.effective_chat.id,
text=caption)
threading.Thread(target=shutdown).start()
return True
elif "Lo sentimos, el código introducido no es correcto" in resp_text:
ctx.bot.send_message(chat_id=update.effective_chat.id,
text="Incorrect, please try again")
else:
error_name = f"error-{ctime}.png".replace(":", "-")
driver.save_screenshot(error_name)
ctx.bot.send_photo(
chat_id=update.effective_chat.id,
photo=open(os.path.join(os.getcwd(), error_name),
"rb"),
)
ctx.bot.send_message(chat_id=update.effective_chat.id,
text="Something went wrong")
dispatcher.add_handler(
CommandHandler("code", code_received, pass_args=True))
context.updater.start_polling(poll_interval=1.0)
for i in range(5):
speaker.say("ALARM")
# Waiting for response 5 minutes
time.sleep(360)
threading.Thread(target=shutdown).start()
if context.save_artifacts:
driver.save_screenshot(
f"FINAL-SCREEN-{datetime.datetime.now()}.png".replace(
":", "-"))
if context.bot_result:
driver.quit()
os._exit(0)
return None
else:
for i in range(5):
speaker.say("ALARM")
logging.info("Press Any button to CLOSE browser")
input()
else:
logging.info("Cita attempt -> missed confirmation :(")
if context.save_artifacts:
driver.save_screenshot(
f"failed-confirmation-{datetime.datetime.now()}.png".replace(
":", "-"))
return None
