class Browser:
driver = None
def __init__(self):
capabilities = DesiredCapabilities.CHROME
# capabilities["loggingPrefs"] = {"performance": "ALL"} # chromedriver < ~75
capabilities["goog:loggingPrefs"] = {
"performance": "ALL"
} # chromedriver 75+
self.driver = Chrome(CHROMEDRV_PATH, desired_capabilities=capabilities)
def goto(self, url):
self.driver.get(url)
def set_input_value(self, name, value):
self.driver.find_element_by_id(name).send_keys(value)
def click_link(self, class_name):
self.driver.find_element_by_class_name(class_name).click()
def check_elem_exist(self, name):
elem = None
try:
elem = self.driver.find_element_by_xpath(f"//*[text()='{name}']")
finally:
return elem
def quit(self):
self.driver.quit()
def get_web_driver(url, username, password):
driver = Chrome()
driver.get(url)
driver.implicitly_wait(1) # seconds
driver.find_element_by_id("loginuser").send_keys(username)
driver.find_element_by_id("loginpass").send_keys(password)
return driver
def get_web_driver(url, username, password):
driver = Chrome()
driver.get(url)
driver.implicitly_wait(1) # seconds
driver.find_element_by_id("ctl00_Main_LoginBox_UserName").send_keys(
username)
driver.find_element_by_id("ctl00_Main_LoginBox_Password").send_keys(
password)
return driver
def get_web_driver(url, username, password):
driver = Chrome()
driver.get(url)
driver.implicitly_wait(1) # seconds
driver.switch_to_frame("login_page")
driver.find_element_by_name("fldLoginUserId").send_keys(username)
driver.implicitly_wait(3)
driver.find_element_by_id('chkrsastu').click()
driver.find_element_by_name("fldPassword").send_keys(password)
driver.implicitly_wait(5)
return driver
def login():
driver = Chrome(driver_path)
driver.get(config.url_login)
login_data = read_secret()
for k in login_data.keys():
box = driver.find_element_by_id(k)
box.send_keys(login_data[k])
time.sleep(config.sleep_short)
# login!
btn = driver.find_element_by_id('login_btn')
btn.click()
time.sleep(config.sleep_long)
# dump the cookies
pickle.dump(driver.get_cookies(), open(cookie_path, "wb"))
def login(user_email_address, user_password, user_profile_url):
"""
user_email_address = str Your Email
user_password = str Your password
user_profile_url = str Your profile URL
"""
chrome_options = Options()
prefs = {
"profile.default_content_setting_values.notifications": 2,
'disk-cache-size': 4096
}
chrome_options.add_experimental_option("prefs", prefs)
chrome_options.add_argument("start-maximized")
driver = Chrome(options=chrome_options)
driver.implicitly_wait(10)
driver.get("https://facebook.com")
email = "email"
password = "******"
login = "******"
emailelement = driver.find_element_by_name(email)
passwordelement = driver.find_element_by_name(password)
emailelement.send_keys(user_email_address)
passwordelement.send_keys(user_password)
loginelement = driver.find_element_by_id(login)
loginelement.click()
driver.get(user_profile_url + '/friends')
return driver
def delete_posts(user_email_address=None,
user_password=None,
user_profile_url=None):
"""
user_email_address: Your Email
user_password: Your password
user_profile_url: Your profile URL
"""
assert all((user_email_address,
user_password,
user_profile_url)), "Insufficient arguments provided"
chrome_options = Options()
prefs = {"profile.default_content_setting_values.notifications" : 2}
chrome_options.add_experimental_option("prefs", prefs)
chrome_options.add_argument("start-maximized")
driver = Chrome(chrome_options=chrome_options)
driver.implicitly_wait(10)
driver.get("https://facebook.com")
email = "email"
password = "******"
login="******"
emailelement = driver.find_element_by_name(email)
passwordelement = driver.find_element_by_name(password)
emailelement.send_keys(user_email_address)
passwordelement.send_keys(user_password)
loginelement = driver.find_element_by_id(login)
loginelement.click()
driver.get(user_profile_url)
for _ in range(MAX_POSTS):
post_button_sel = "_4xev"
timeline_element = driver.find_element_by_class_name(post_button_sel)
actions = ActionChains(driver)
actions.move_to_element(timeline_element).click().perform()
menu = driver.find_element_by_css_selector("#globalContainer > div.uiContextualLayerPositioner.uiLayer > div")
actions.move_to_element(menu).perform()
try:
delete_button = menu.find_element_by_xpath("//a[@data-feed-option-name=\"FeedDeleteOption\"]")
except:
delete_button = menu.find_element_by_xpath("//a[@data-feed-option-name=\"HIDE_FROM_TIMELINE\"]")
actions.move_to_element(delete_button).click().perform()
confirmation_button = driver.find_element_by_class_name("layerConfirm")
driver.execute_script("arguments[0].click();", confirmation_button)
time.sleep(5)
driver.refresh()
def get_web_driver(email, password):
driver = Chrome()
driver.get("https://www.mint.com")
driver.implicitly_wait(20) # seconds
driver.find_element_by_link_text("Log In").click()
driver.find_element_by_id("ius-userid").send_keys(email)
driver.find_element_by_id("ius-password").send_keys(password)
driver.find_element_by_id("ius-sign-in-submit-btn").submit()
# Wait until logged in, just in case we need to deal with MFA.
while not driver.current_url.startswith(
'https://mint.intuit.com/overview.event'):
time.sleep(1)
# Wait until the overview page has actually loaded.
driver.implicitly_wait(20) # seconds
driver.find_element_by_id("transaction")
return driver
def get_web_driver(email, password):
driver = Chrome()
driver.get("https://www.mint.com")
driver.implicitly_wait(20) # seconds
driver.find_element_by_link_text("Log In").click()
driver.find_element_by_id("ius-userid").send_keys(email)
driver.find_element_by_id("ius-password").send_keys(password)
driver.find_element_by_id("ius-sign-in-submit-btn").submit()
# Wait until logged in, just in case we need to deal with MFA.
while not driver.current_url.startswith(
'https://mint.intuit.com/overview.event'):
time.sleep(1)
# Wait until the overview page has actually loaded.
driver.implicitly_wait(20) # seconds
driver.find_element_by_id("transaction")
return driver
def login(user_email_address, user_password, user_profile_url, is_headless,
two_factor_token):
"""
Attempts to log into Facebook
Returns a driver object
user_email_address: str Your Email
user_password: str Your password
user_profile_url: str Your profile URL
"""
# The Chrome driver is required because Gecko was having issues
chrome_options = Options()
prefs = {
"profile.default_content_setting_values.notifications": 2,
'disk-cache-size': 4096
}
chrome_options.add_experimental_option("prefs", prefs)
chrome_options.add_argument("start-maximized")
if is_headless:
chrome_options.add_argument('--headless')
chrome_options.add_argument('--disable-gpu')
chrome_options.add_argument('log-level=2')
driver = Chrome(options=chrome_options)
driver.implicitly_wait(10)
driver.get("https://facebook.com")
email = "email"
password = "******"
login = "******"
approvals_code = "approvals_code"
emailelement = driver.find_element_by_name(email)
passwordelement = driver.find_element_by_name(password)
emailelement.send_keys(user_email_address)
passwordelement.send_keys(user_password)
loginelement = driver.find_element_by_id(login)
loginelement.click()
if "two-factor authentication" in driver.page_source.lower():
if two_factor_token:
twofactorelement = driver.find_element_by_name(approvals_code)
twofactorelement.send_keys(two_factor_token)
# Submits after the code is passed into the form, does not validate 2FA code.
contelement = driver.find_element_by_id("checkpointSubmitButton")
contelement.click()
# Defaults to saving this new browser, this occurs on each new automated login.
save_browser = driver.find_element_by_id("checkpointSubmitButton")
save_browser.click()
else:
# Allow time to enter 2FA code
print("Pausing to enter 2FA code")
time.sleep(20)
print("Continuing execution")
driver.get(user_profile_url)
return driver
# browser authentication for auth and cookie grabbing
load_dotenv()
PROJECT_ROOT = os.path.abspath(os.path.dirname(__file__))
DRIVER_BIN = os.path.join(PROJECT_ROOT, "bin/chromedriver")
UW_EMAIL=os.getenv("UW_EMAIL")
UW_NETID=os.getenv("UW_NETID")
UW_PASSWORD=os.getenv("UW_PASSWORD")
driver = Chrome(executable_path = DRIVER_BIN)
driver.set_window_position(0, 0)
driver.set_window_size(400, 1000)
driver.get("http://localhost:3000/api/login")
driver.find_element_by_id("identifierId").send_keys(UW_EMAIL)
driver.find_element_by_id("identifierNext").click()
time.sleep(1)
driver.find_element_by_id("weblogin_netid").send_keys(UW_NETID)
driver.find_element_by_id("weblogin_password").send_keys(UW_PASSWORD)
driver.find_element_by_id("submit_button").click()
time.sleep(1)
driver.find_element_by_xpath('//button[normalize-space()="Continue"]').click()
time.sleep(1)
driver.find_element_by_xpath('//button[normalize-space()="New referral"]').click()
# start for a browser automation approach
# CREATE A CHILD
driver.find_element_by_id("childFirstName").send_keys('Cindy')
driver.find_element_by_id("childLastName").send_keys('Martin')
def get_web_driver(email, password, code_provider):
driver = Chrome()
driver.get("https://www.mint.com")
driver.implicitly_wait(20) # seconds
driver.find_element_by_link_text("Log In").click()
driver.find_element_by_id("ius-userid").send_keys(email)
driver.find_element_by_id("ius-password").send_keys(password)
driver.find_element_by_id("ius-sign-in-submit-btn").submit()
# Wait until logged in, just in case we need to deal with MFA.
while not driver.current_url.startswith(
'https://mint.intuit.com/overview.event'):
time.sleep(2)
try:
mfa_ele = driver.find_element_by_id("ius-mfa-option-email")
mfa_ele.click()
driver.find_element_by_id("ius-mfa-options-submit-btn").click()
verification_code = code_provider()
driver.find_element_by_id("ius-mfa-confirm-code").send_keys(
verification_code)
driver.find_element_by_id("ius-mfa-otp-submit-btn").submit()
except:
pass
# Wait until the overview page has actually loaded.
driver.implicitly_wait(20) # seconds
driver.find_element_by_id("transaction")
return driver
def get_amzn_driver(email, password, headless=False, session_path=None):
chrome_options = ChromeOptions()
if headless:
chrome_options.add_argument('headless')
chrome_options.add_argument('no-sandbox')
chrome_options.add_argument('disable-dev-shm-usage')
chrome_options.add_argument('disable-gpu')
if session_path is not None:
chrome_options.add_argument("user-data-dir=" + session_path)
logger.info('Logging into Amazon.com')
home_dir = os.path.expanduser("~")
driver = Chrome(options=chrome_options,
executable_path=get_stable_chrome_driver(home_dir))
driver.get(ORDER_HISTORY_URL_VIA_SWITCH_ACCOUNT_LOGIN)
driver.implicitly_wait(2)
def get_element_by_id(driver, id):
try:
return driver.find_element_by_id(id)
except NoSuchElementException:
pass
return None
def get_element_by_xpath(driver, xpath):
try:
return driver.find_element_by_xpath(xpath)
except NoSuchElementException:
pass
return None
# Go straight to the account switcher, and look for the given email.
# If present, click on it! Otherwise, click on "Add account".
desired_account_element = get_element_by_xpath(
driver, "//div[contains(text(), '{}')]".format(email))
if desired_account_element:
desired_account_element.click()
driver.implicitly_wait(2)
# It's possible this account has already authed recently. If so, the
# next block will be skipped and the login is complete!
if not get_element_by_id(driver, 'report-confirm'):
driver.find_element_by_id('ap_password').send_keys(
get_password(password))
driver.find_element_by_name('rememberMe').click()
driver.find_element_by_id('signInSubmit').submit()
else:
# Cannot find the desired account in the switch. Log in via Add Account
driver.find_element_by_xpath('//div[text()="Add account"]').click()
driver.implicitly_wait(2)
driver.find_element_by_id('ap_email').send_keys(email)
# Login flow sometimes asks just for the email, then a
# continue button, then password.
if get_element_by_id(driver, 'continue'):
driver.find_element_by_id('continue').click()
driver.implicitly_wait(2)
driver.find_element_by_id('ap_password').send_keys(
get_password(password))
driver.find_element_by_name('rememberMe').click()
driver.find_element_by_id('signInSubmit').submit()
driver.implicitly_wait(2)
if not get_element_by_id(driver, 'report-confirm'):
logger.warning('Having trouble logging into Amazon. Please see the '
'browser and complete login within the next 5 minutes. '
'This script will continue automatically on success. '
'You may need to manually navigate to: {}'.format(
ORDER_HISTORY_REPORT_URL))
if get_element_by_id(driver, 'auth-mfa-otpcode'):
logger.warning('Hint: Looks like an auth challenge! Maybe check '
'your email')
try:
wait_cond = EC.presence_of_element_located((By.ID, 'report-confirm'))
WebDriverWait(driver, 60 * 5).until(wait_cond)
except TimeoutException:
logger.critical('Cannot complete login!')
exit(1)
return driver
class UmsConnection():
def __init__(self, user_phone, user_password, cookies=None):
self._user_phone = user_phone
self._user_password = user_password
self._is_authorized = False
self._captcha = None # REMOVE
options = Options()
options.add_argument("--headless")
WINDOW_SIZE = "1620,880" # ?
options.add_argument("--window-size=%s" % WINDOW_SIZE)
self.webdriver = Chrome(chrome_options=options)
if cookies:
self.webdriver.get('https://messages.megafon.ru/onebox/mix.do')
for cookie in cookies:
self.webdriver.add_cookie(cookie)
self.webdriver.get('https://messages.megafon.ru/onebox/mix.do')
self._is_authorized = self.try_is_authorized()
if not self.is_authorized:
self.webdriver.get(r"https://messages.megafon.ru/onebox/mix.do")
phone_input = self.webdriver.find_element_by_id('user12')
phone_input.send_keys(user_phone)
password_input_secinfo = self.webdriver.find_element_by_id('secinfo')
password_input_secinfo.click()
password_input = self.webdriver.find_element_by_id("secinfo2")
password_input.send_keys(user_password)
def get_captcha(self) -> bytes:
captcha = self.webdriver.find_element_by_id('imageC')
location = captcha.location
size = captcha.size
screenshot = self.webdriver.get_screenshot_as_png()
im = Image.open(BytesIO(screenshot))
left = location['x']
top = location['y']
right = location['x'] + size['width']
bottom = location['y'] + size['height']
im = im.crop((left, top, right, bottom))
self._captcha = im # REMOVE
stream = BytesIO()
im.save(stream, format="PNG")
return stream.getvalue()
def send_captcha_key(self, key) -> bool:
captcha_input = self.webdriver.find_element_by_id('imageObj')
captcha_input.send_keys(key)
submit_button = self.webdriver.find_element_by_id('index_login')
submit_button.click()
try:
WebDriverWait(self.webdriver, 2).until(
EC.url_contains(("messages.megafon.ru/onebox/mix.do")))
self._is_authorized = True
self._captcha.save(f'captchas/{key}.png') # REMOVE
return True
except exceptions.TimeoutException:
return False
def try_is_authorized(self) -> bool:
# TODO
url = r"https://messages.megafon.ru/onebox/getChatList.do"
get_chats_url_params = {
'startNum': '1',
'endNum': 10,
'reFreshFlag': '1',
'operation': '1',
'chatMsgType': '10100000000110000000000000000000',
't': random.random()
}
response = self.webdriver.request('GET', url, params=get_chats_url_params)
return response.status_code == 200
@property
def is_authorized(self):
return self._is_authorized
def get_chat_list(self, number=10):
url = r"https://messages.megafon.ru/onebox/getChatList.do"
get_chats_url_params = {
'startNum': '1',
'endNum': number,
'reFreshFlag': '1',
'operation': '1',
'chatMsgType': '10100000000110000000000000000000',
't': random.random()
}
response = self.webdriver.request('GET', url, params=get_chats_url_params)
return response.text
def get_one_box(self, address, number=10, from_num=1):
url = r"https://messages.megafon.ru/onebox/oneboxList.do"
params = [
('umReq.ctlg', '1,2'),
('umReq.numFlg', '1'),
('umReq.mType', '6149'),
('umReq.srt', '0'),
('umReq.lType', '0'),
('umReq.dFlg', '0'),
('umReq.srtDr', '0'),
('umReq.rdFlg', '0'),
('umReq.bNum', from_num),
('umReq.eNum', number),
('umReq.snd', address),
('umReq.rcv', self._user_phone),
('umReq.bTime', ''),
('umReq.eTime', ''),
('umReq.impt', '-1'),
('umReq.t', ''),
('umReq.threadFlag', '1'),
('rownid', random.random()),
]
response = self.webdriver.request('GET', url, params=params)
return response.text
def close(self):
self.webdriver.close()
def get_cookies_json(self):
cookies = self.webdriver.get_cookies()
return json.dumps(cookies)
class TescoScraper:
email = os.environ.get('tesco_email')
pw = os.environ.get('tesco_password')
twilio_auth = os.environ.get('twilio_auth')
locations = {'7268': "Banbridge",
'7615': 'Craigavon',
'7214': 'Lisburn',
'7275': 'Newry'}
phone_numbers = os.environ.get('phone_numbers').split(",")
print(email)
print(phone_numbers)
login_page = "https://secure.tesco.com/account/en-GB/login"
delivery_url_with_date_and_slot_group = "https://www.tesco.com/groceries/en-GB/slots/delivery/%s?slotGroup=%s"
collection_url_with_date = "https://www.tesco.com/groceries/en-GB/slots/collection/%s?locationId=%s&postcode=&slotGroup=4"
unavail_status = ["Unavailable", "UnAvailable", "UNAvailable"]
driver = None
def setupSelenium(self):
CHROMEDRIVER_PATH = '/app/.chromedriver/bin/chromedriver'
chrome_options = Options()
chrome_options.binary_location = os.environ.get('GOOGLE_CHROME_SHIM')
self.driver = Chrome(executable_path=CHROMEDRIVER_PATH, options=chrome_options)
chrome_options.add_argument("--headless")
chrome_options.add_argument("--disable-gpu")
chrome_options.add_argument("--window-size=1920,1080")
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument(
'--user-agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.92 Safari/537.36"')
def sendEmail(self, location: str, date):
send_grid_key = os.environ.get('send_grid')
message = Mail(
from_email='*****@*****.**',
to_emails='*****@*****.**',
subject=f'Tesco Delivery slot available {location} at {date}',
html_content=f'<strong>Avail in {location} at {date}</strong>')
try:
sg = SendGridAPIClient(send_grid_key)
response = sg.send(message)
print(response.status_code)
print(response.body)
print(response.headers)
except Exception as e:
print(e)
def sendTextMessage(self, collectionOrDelivery, location, date, button_details):
if os.environ.get('text_enabled') == '0':
return
account_sid = os.environ.get('twilio_id')
client = Client(account_sid, self.twilio_auth)
text_message = collectionOrDelivery + " Tesco Slot Available @ " + location + " at date " + button_details
print(collectionOrDelivery + text_message + str(datetime.now()))
for number in self.phone_numbers:
message = client.messages \
.create(
body=text_message,
from_='+13103214290',
to=number
)
print("Sent notification message " + text_message)
exit(1)
def is_logged_id(self):
self.driver.get(self.login_page)
try:
slot_message = WebDriverWait(self.driver, 10).until(
lambda driver: self.driver.find_element_by_id("username"))
return False
except:
return True
def loginToTesco(self):
self.driver.get(self.login_page)
username = self.driver.find_element_by_id("username")
username.clear()
username.send_keys(self.email)
password = self.driver.find_element_by_id("password")
password.clear()
password.send_keys(self.pw)
buttons = WebDriverWait(self.driver, 10).until(
lambda driver: driver.find_elements_by_class_name("ui-component__button"))
for b in buttons:
b.click()
time.sleep(1)
time.sleep(5)
# print("Login Complete")
def scanForSlots(self):
print("scanning " + str(datetime.now()))
today = (datetime.now()).strftime('%Y-%m-%d')
next_week = (datetime.now() + timedelta(days=7)).strftime('%Y-%m-%d')
fortnite = (datetime.now() + timedelta(days=14)).strftime('%Y-%m-%d')
self.driver.get("https://www.tesco.com/groceries/en-GB/slots/collection")
for start_date in [today, next_week, fortnite]:
# Scan 3 week window
for tesco_location in self.locations:
# try to search for collection slots first of all
grocery_collection_url = self.collection_url_with_date % (start_date, tesco_location)
try:
self.driver.get(grocery_collection_url)
except Exception as e:
print("Exception caught from delivery load")
return
try:
buttons = WebDriverWait(self.driver, 5).until(
lambda driver: driver.find_elements_by_class_name("available-slot--button"))
buttons = [b for b in buttons if b.text is not '']
if len(buttons) > 0:
print("Collection for " + start_date + " at " + self.locations[tesco_location])
[print(b.txext) for b in buttons]
print()
if os.environ.get('auto_book_enabled') == '1':
buttons[-1].click()
button_details = buttons[-1].text
self.sendTextMessage("Home Delivery", "Delivery", start_date, button_details)
except:
var = True
# nothing found
try:
self.driver.get(self.delivery_url_with_date_and_slot_group % (start_date, 1))
except Exception as e:
print("Exception caught from delivery load at slow group")
print(e)
print("Invalid session not sure why")
return
# try to search for delivery slots next
try:
buttons = WebDriverWait(self.driver, 5).until(lambda driver: driver.find_elements_by_class_name("available-slot--button"))
buttons = [b for b in buttons if b.text is not '']
if len(buttons) > 0:
print("Home delivery for " + start_date)
[print(b.txext) for b in buttons]
print()
if os.environ.get('auto_book_enabled') == '1':
buttons[-1].click()
button_details = buttons[-1].text
self.sendTextMessage("Home Delivery", "Delivery", start_date, button_details)
except:
var = True
try:
self.driver.get(self.delivery_url_with_date_and_slot_group % (start_date, 4))
except Exception as e:
print("Exception caught from delivery load")
print(e)
print("Invalid session not sure why")
return
# try to search for delivery slots next
try:
buttons = WebDriverWait(self.driver, 5).until(lambda driver: driver.find_elements_by_class_name("available-slot--button"))
buttons = [b for b in buttons if b.text is not '']
if len(buttons) > 0:
print("Home delivery for " + start_date)
[print(b.txext) for b in buttons]
print()
if os.environ.get('auto_book_enabled') == '1':
buttons[-1].click()
button_details = buttons[-1].text
self.sendTextMessage("Home Delivery", "Delivery", start_date, button_details)
except:
var = True
def delete_posts(user_email_address=None,
user_password=None,
user_profile_url=None):
"""
user_email_address: Your Email
user_password: Your password
user_profile_url: Your profile URL
"""
assert all((user_email_address, user_password,
user_profile_url)), "Insufficient arguments provided"
# The Chrome driver is required because Gecko was having issues
chrome_options = Options()
prefs = {"profile.default_content_setting_values.notifications": 2}
chrome_options.add_experimental_option("prefs", prefs)
chrome_options.add_argument("start-maximized")
driver = Chrome(chrome_options=chrome_options)
driver.implicitly_wait(10)
driver.get("https://facebook.com")
email = "email"
password = "******"
login = "******"
emailelement = driver.find_element_by_name(email)
passwordelement = driver.find_element_by_name(password)
emailelement.send_keys(user_email_address)
passwordelement.send_keys(user_password)
loginelement = driver.find_element_by_id(login)
loginelement.click()
if "Two-factor authentication" in driver.page_source:
# Allow time to enter 2FA code
print("Pausing to enter 2FA code")
time.sleep(20)
print("Continuing execution")
driver.get(user_profile_url)
for _ in range(MAX_POSTS):
post_button_sel = "_4xev"
timeline_element = driver.find_element_by_class_name(post_button_sel)
actions = ActionChains(driver)
actions.move_to_element(timeline_element).click().perform()
menu = driver.find_element_by_css_selector(
"#globalContainer > div.uiContextualLayerPositioner.uiLayer > div")
actions.move_to_element(menu).perform()
try:
delete_button = menu.find_element_by_xpath(
"//a[@data-feed-option-name=\"FeedDeleteOption\"]")
# FIXME Using a bare except here to avoid having to handle all possible exceptions
except:
delete_button = menu.find_element_by_xpath(
"//a[@data-feed-option-name=\"HIDE_FROM_TIMELINE\"]")
actions.move_to_element(delete_button).click().perform()
confirmation_button = driver.find_element_by_class_name("layerConfirm")
# Facebook would not let me get focus on this button without some custom JS
driver.execute_script("arguments[0].click();", confirmation_button)
# Required to sleep the thread for a bit after using JS to click this button
time.sleep(5)
driver.refresh()
def delete_posts(user_email_address, user_password, user_profile_url,
is_headless):
"""
user_email_address: str Your Email
user_password: str Your password
user_profile_url: str Your profile URL
"""
# The Chrome driver is required because Gecko was having issues
chrome_options = Options()
prefs = {
"profile.default_content_setting_values.notifications": 2,
'disk-cache-size': 4096
}
chrome_options.add_experimental_option("prefs", prefs)
chrome_options.add_argument("start-maximized")
if is_headless:
chrome_options.add_argument('--headless')
chrome_options.add_argument('--disable-gpu')
chrome_options.add_argument('log-level=2')
driver = Chrome(options=chrome_options)
driver.implicitly_wait(10)
driver.get("https://facebook.com")
email = "email"
password = "******"
login = "******"
emailelement = driver.find_element_by_name(email)
passwordelement = driver.find_element_by_name(password)
emailelement.send_keys(user_email_address)
passwordelement.send_keys(user_password)
loginelement = driver.find_element_by_id(login)
loginelement.click()
if "Two-factor authentication" in driver.page_source:
# Allow time to enter 2FA code
print("Pausing to enter 2FA code")
time.sleep(20)
print("Continuing execution")
driver.get(user_profile_url)
for _ in range(MAX_POSTS):
post_button_sel = "_4xev"
while True:
try:
timeline_element = driver.find_element_by_class_name(
post_button_sel)
actions = ActionChains(driver)
actions.move_to_element(timeline_element).click().perform()
menu = driver.find_element_by_css_selector(
"#globalContainer > div.uiContextualLayerPositioner.uiLayer > div"
)
actions.move_to_element(menu).perform()
try:
delete_button = menu.find_element_by_xpath(
"//a[@data-feed-option-name=\"FeedDeleteOption\"]")
except SELENIUM_EXCEPTIONS:
delete_button = menu.find_element_by_xpath(
"//a[@data-feed-option-name=\"HIDE_FROM_TIMELINE\"]")
actions.move_to_element(delete_button).click().perform()
confirmation_button = driver.find_element_by_class_name(
"layerConfirm")
# Facebook would not let me get focus on this button without some custom JS
driver.execute_script("arguments[0].click();",
confirmation_button)
except SELENIUM_EXCEPTIONS:
continue
else:
break
# Required to sleep the thread for a bit after using JS to click this button
time.sleep(5)
driver.refresh()
def get_web_driver(email, password, headless=False, mfa_method=None,
mfa_input_callback=None, wait_for_sync=True,
session_path=None, imap_account=None, imap_password=None,
imap_server=None, imap_folder="INBOX"):
if headless and mfa_method is None:
warnings.warn("Using headless mode without specifying an MFA method"
"is unlikely to lead to a successful login. Defaulting --mfa-method=sms")
mfa_method = "sms"
zip_type = ""
executable_path = os.getcwd() + os.path.sep + 'chromedriver'
if _platform in ['win32', 'win64']:
executable_path += '.exe'
zip_type = CHROME_ZIP_TYPES.get(_platform)
if not os.path.exists(executable_path):
zip_file_url = CHROME_DRIVER_BASE_URL % (CHROME_DRIVER_VERSION, zip_type)
request = requests.get(zip_file_url)
if request.status_code != 200:
raise RuntimeError('Error finding chromedriver at %r, status = %d' %
(zip_file_url, request.status_code))
zip_file = zipfile.ZipFile(io.BytesIO(request.content))
zip_file.extractall()
os.chmod(executable_path, 0o755)
chrome_options = ChromeOptions()
if headless:
chrome_options.add_argument('headless')
chrome_options.add_argument('no-sandbox')
chrome_options.add_argument('disable-dev-shm-usage')
chrome_options.add_argument('disable-gpu')
# chrome_options.add_argument("--window-size=1920x1080")
if session_path is not None:
chrome_options.add_argument("user-data-dir=%s" % session_path)
driver = Chrome(chrome_options=chrome_options, executable_path="%s" % executable_path)
driver.get("https://www.mint.com")
driver.implicitly_wait(20) # seconds
try:
element = driver.find_element_by_link_text("Log In")
except NoSuchElementException:
# when user has cookies, a slightly different front page appears
driver.implicitly_wait(0) # seconds
element = driver.find_element_by_link_text("LOG IN")
driver.implicitly_wait(20) # seconds
element.click()
time.sleep(1)
driver.find_element_by_id("ius-userid").send_keys(email)
driver.find_element_by_id("ius-password").send_keys(password)
driver.find_element_by_id("ius-sign-in-submit-btn").submit()
# Wait until logged in, just in case we need to deal with MFA.
while not driver.current_url.startswith(
'https://mint.intuit.com/overview.event'):
time.sleep(1)
driver.implicitly_wait(1) # seconds
try:
driver.find_element_by_id('ius-mfa-options-form')
try:
mfa_method_option = driver.find_element_by_id('ius-mfa-option-{}'.format(mfa_method))
mfa_method_option.click()
mfa_method_submit = driver.find_element_by_id("ius-mfa-options-submit-btn")
mfa_method_submit.click()
if mfa_method == 'email' and imap_account:
mfa_code = get_email_code(imap_account, imap_password, imap_server, imap_folder=imap_folder)
else:
mfa_code = (mfa_input_callback or input)("Please enter your 6-digit MFA code: ")
mfa_code_input = driver.find_element_by_id("ius-mfa-confirm-code")
mfa_code_input.send_keys(mfa_code)
mfa_code_submit = driver.find_element_by_id("ius-mfa-otp-submit-btn")
mfa_code_submit.click()
except Exception: # if anything goes wrong for any reason, give up on MFA
mfa_method = None
warnings.warn("Giving up on handling MFA. Please complete "
"the MFA process manually in the browser.")
except NoSuchElementException:
pass
finally:
driver.implicitly_wait(20) # seconds
# Wait until the overview page has actually loaded, and if wait_for_sync==True, sync has completed.
if wait_for_sync:
status_message = driver.find_element_by_css_selector(".SummaryView .message")
try:
WebDriverWait(driver, 5 * 60).until(
lambda x: "Account refresh complete" in status_message.get_attribute('innerHTML')
)
except TimeoutException:
warnings.warn("Mint sync apparently incomplete after 5 minutes. Data "
"retrieved may not be current.")
else:
driver.find_element_by_id("transaction")
return driver
# def __contains__(self, key):
# return key in self.keys()
# def __iter__(self):
# return self.items().__iter__()
# def __repr__(self):
# return self.items().__str__()
driver = Chrome(r"C:/Python/deezLoaderScript/chromedriver/chromedriver_win32/chromedriver.exe")
payload = {"login_mail":"*****@*****.**", "login_password":"******"}
pathToAPI = r"C:/Music/Deezloader-win32-x64/resources/app/deezer-api.js"
pathToDeezer = r"C:/Music/Deezloader-win32-x64/Deezloader.exe"
url='https://www.deezer.com/login/email'
driver.get(url)
user = driver.find_element_by_id("login_mail")
password = driver.find_element_by_id("login_password")
submit = driver.find_element_by_id("login_form_submit")
user.send_keys(payload["login_mail"])
password.send_keys(payload["login_password"])
submit.click()
r = driver.request("POST",url,data=payload)
j = str(r.headers)
js = ast.literal_eval(j)
sid = js["Set-Cookie"].split(";")[0]
print(sid)
changeStr = '\t\t"Cookie":"'+sid+'"'
lines = []
lineIndex = 0
with open(pathToAPI) as f:
chrome_options.add_argument('--data-path=/tmp/data-path')
chrome_options.add_argument('--homedir=/tmp')
chrome_options.add_argument('--disk-cache-dir=/tmp/cache-dir')
prefs={"profile.managed_default_content_settings.images": 2, 'disk-cache-size': 4096 }
chrome_options.add_experimental_option("prefs",prefs)
delay = 60
chrome_options.binary_location = "/usr/bin/chromium-browser"
driver = Chrome(executable_path=os.path.abspath("/usr/lib/chromium-browser/chromedriver"), chrome_options=chrome_options)
print("logging into outlook")
driver.get("https://outlook.office.com/owa/?ae=Folder&t=IPF.Appointment")
try:
driver.find_element_by_name("loginfmt").send_keys("*****@*****.**")
driver.find_element_by_id("idSIButton9").click()
print("entered username, waiting for password prompt")
try:
myElem = WebDriverWait(driver, delay).until(EC.presence_of_element_located((By.ID, 'branding')))
print("password prompt loaded")
except TimeoutException:
print("Loading password prompt took too much time!")
driver.close();
exit(1)
passwd = getpass.getpass()
driver.find_element_by_id("passwordInput").send_keys(passwd)
driver.find_element_by_id("submitButton").click()
print("entered password, waiting for 2FA token")
def test_basic_training():
Config.init_config('../app_conf/testing.ini')
chrome_options = Options()
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--headless')
chrome_options.add_argument("--disable-user-media-security")
chrome_options.add_argument("--use-fake-ui-for-media-stream")
chrome_options.add_argument("--use-fake-device-for-media-stream")
chrome_options.add_argument(
'--use-file-for-fake-audio-capture={}/simple_phrases_russian.wav'.
format(os.getcwd()))
chrome_options.add_experimental_option('detach', True)
driver = Chrome(options=chrome_options)
response = driver.request('POST',
'http://127.0.0.1:5000/lti',
data={
'lis_person_name_full':
Config.c.testing.lis_person_name_full,
'ext_user_username':
Config.c.testing.session_id,
'custom_task_id':
Config.c.testing.custom_task_id,
'custom_task_description':
Config.c.testing.custom_task_description,
'custom_attempt_count':
Config.c.testing.custom_attempt_count,
'custom_required_points':
Config.c.testing.custom_required_points,
'custom_criteria_pack_id':
Config.c.testing.custom_criteria_pack_id,
'roles':
Config.c.testing.roles,
'lis_outcome_service_url':
Config.c.testing.lis_outcome_service_url,
'lis_result_sourcedid':
Config.c.testing.lis_result_source_did,
'oauth_consumer_key':
Config.c.testing.oauth_consumer_key,
})
driver.get('http://127.0.0.1:5000/upload_presentation/')
driver.find_element_by_id('upload-presentation-form')
data = open('test_data/test_presentation_file_0.pdf', 'rb')
response = driver.request(
'POST',
'http://127.0.0.1:5000/handle_presentation_upload/',
files=dict(presentation=data))
pos = response.text.find("setupPresentationViewer(\"")
assert pos != -1
training_id = response.text[pos + 25:pos + 49]
driver.get('http://127.0.0.1:5000/trainings/{}/'.format(training_id))
driver.find_element_by_id('record').click()
step = 3
sleep(2 * step)
driver.find_element_by_id('next').click()
sleep(step)
driver.find_element_by_id('done').click()
sleep(step)
total_wait_time = 60
wait_time = 0
while wait_time < total_wait_time:
driver.get('http://127.0.0.1:5000/trainings/statistics/{}/'.format(
training_id))
try:
feedback_element = WebDriverWait(driver, step).until(
EC.presence_of_element_located((By.ID, 'feedback')))
if feedback_element.text.startswith('Оценка за тренировку'):
break
else:
wait_time += step
sleep(step)
except TimeoutException:
wait_time += step
driver.close()
assert wait_time < total_wait_time
def get_web_driver(email,
password,
headless=False,
mfa_method=None,
mfa_input_callback=None,
wait_for_sync=True):
if headless and mfa_method is None:
warnings.warn(
"Using headless mode without specifying an MFA method"
"is unlikely to lead to a successful login. Defaulting --mfa-method=sms"
)
mfa_method = "sms"
zip_type = ""
executable_path = os.getcwd() + os.path.sep + 'chromedriver'
if _platform in ['win32', 'win64']:
executable_path += '.exe'
zip_type = CHROME_ZIP_TYPES.get(_platform)
if not os.path.exists(executable_path):
zip_file_url = CHROME_DRIVER_BASE_URL % (CHROME_DRIVER_VERSION,
zip_type)
request = requests.get(zip_file_url)
if request.status_code != 200:
raise RuntimeError(
'Error finding chromedriver at %r, status = %d' %
(zip_file_url, request.status_code))
zip_file = zipfile.ZipFile(io.BytesIO(request.content))
zip_file.extractall()
os.chmod(executable_path, 0o755)
chrome_options = ChromeOptions()
if headless:
chrome_options.add_argument('headless')
chrome_options.add_argument('no-sandbox')
chrome_options.add_argument('disable-dev-shm-usage')
chrome_options.add_argument('disable-gpu')
# chrome_options.add_argument("--window-size=1920x1080")
driver = Chrome(chrome_options=chrome_options,
executable_path="%s" % executable_path)
driver.get(
"https://accounts.intuit.com/index.html?redirect_url=https%3A%2F%2Fmint.intuit.com%2Foverview.event"
)
driver.implicitly_wait(20) # seconds
driver.find_element_by_id("ius-userid").send_keys(email)
driver.find_element_by_id("ius-password").send_keys(password)
driver.find_element_by_id("ius-sign-in-submit-btn").submit()
# Wait until logged in, just in case we need to deal with MFA.
while not driver.current_url.startswith(
'https://mint.intuit.com/overview.event'):
time.sleep(1)
driver.implicitly_wait(1) # seconds
try:
try:
if mfa_method == "app":
# mfa_input_callback should be a function that returns the 2fa barcode.
two_factor_code = mfa_input_callback()
driver.find_element_by_id("ius-mfa-soft-token").send_keys(
two_factor_code)
driver.find_element_by_id(
"ius-mfa-soft-token-submit-btn").submit()
else:
driver.find_element_by_id('ius-mfa-options-form')
mfa_method_option = driver.find_element_by_id(
'ius-mfa-option-{}'.format(mfa_method))
mfa_method_option.click()
mfa_method_submit = driver.find_element_by_id(
"ius-mfa-options-submit-btn")
mfa_method_submit.click()
mfa_code = (mfa_input_callback or
input)("Please enter your 6-digit MFA code: ")
mfa_code_input = driver.find_element_by_id(
"ius-mfa-confirm-code")
mfa_code_input.send_keys(mfa_code)
mfa_code_submit = driver.find_element_by_id(
"ius-mfa-otp-submit-btn")
mfa_code_submit.click()
except NoSuchElementException:
pass
except Exception as e:
mfa_method = None
warnings.warn(
"Giving up on handling MFA. Please complete "
"the MFA process manually in the browser. Exception: " +
repr(e))
except NoSuchElementException:
pass
finally:
driver.implicitly_wait(20) # seconds
# Wait until the overview page has actually loaded, and if wait_for_sync==True, sync has completed.
if wait_for_sync:
try:
driver.implicitly_wait(5)
status_message = driver.find_element_by_css_selector(
".SummaryView .message")
WebDriverWait(driver, 5 *
60).until(lambda x: "Account refresh complete" in
status_message.get_attribute('innerHTML'))
except TimeoutException:
warnings.warn(
"Mint sync apparently incomplete after 5 minutes. Data "
"retrieved may not be current.")
else:
driver.find_element_by_id("transaction")
return driver
class DigitalCommonsConnection:
def __init__(self, user, password):
self.options = Options()
self.driver = Chrome(
executable_path=os.path.abspath("/usr/bin/chromedriver"),
options=self.options)
self.login(user, password)
self.dissertations = self.get_list_of_dissertations()
self.lookup_values = self.__review_dissertations()
self.__lookup_decisions()
def login(self, username, passwd):
self.driver.get(
'https://trace.tennessee.edu/cgi/myaccount.cgi?context=')
self.driver.find_element_by_id('auth_email').send_keys(username)
self.driver.find_element_by_id('auth_password').send_keys(passwd)
self.driver.find_element_by_xpath(
'/html/body/div[2]/div/div[3]/div[1]/div[1]/div/div[2]/div[1]/div/form/div/p/button'
).click()
def get_list_of_dissertations(self):
self.driver.get(
'https://trace.tennessee.edu/utk_graddiss/index.11.html#year_2015')
disserations = self.driver.find_elements_by_css_selector(
'.article-listing > a')
return [
disserations[link].get_attribute('href')
for link in range(0, len(disserations))
]
def __review_dissertations(self):
lookups = []
for dissertation in self.dissertations:
self.driver.get(dissertation)
link = self.driver.find_element_by_css_selector('#title > p > a')
lookups.append(
link.get_attribute('href').split('=')[1].split('&')[0])
return lookups
def __lookup_decisions(self):
for dissertation in self.lookup_values:
self.driver.get(
f'https://trace.tennessee.edu/cgi/editor.cgi?article={dissertation}'
f'&window=viewdecisions&context=utk_graddiss')
decisions = self.driver.find_elements_by_css_selector(
'.MenuMain > tbody > tr > td > table > tbody > tr > td > a')
all_decisions = [
decisions[link].get_attribute('href')
for link in range(0, len(decisions)) if link !=
'https://trace.tennessee.edu/cgi/help.cgi?context=utk_graddiss&help=help-submissions.html#'
]
for decision in all_decisions:
self.driver.get(decision)
try:
final_decision_metadata = self.driver.find_element_by_css_selector(
'.MenuMain > tbody > tr > td > span')
decision_metadata = final_decision_metadata.text
print(decision_metadata.split('\n'))
final_decision = self.driver.find_element_by_css_selector(
'.MenuMain > tbody > tr > td > pre')
except NoSuchElementException:
pass
return
def issue_request():
# region Variable Declarations
global start_date, end_date, \
instrument_type, property_address, \
authenticated
instrument_types = []
recording_dates = []
dated_dates = []
grantors = []
grantees = []
result_page_links = []
property_addresses = []
property_owner_names = []
property_owner_addresses = []
pdf_text = ""
# endregion Variable Declarations
# Browser Instantiation
browser = Chrome(ChromeDriverManager().install())
# The site to which we will navigate while also handling its session.
browser.get(login_url)
# Locates and populates the elements containing the username and password entry fields.
username_element = browser.find_element_by_id("USERID")
username_element.send_keys(username)
password_element = browser.find_element_by_id("PASSWORD")
password_element.send_keys(password)
# Login
password_element.send_keys(Keys.RETURN)
authenticated = True if "invalid" not in str(
browser.page_source) else False
if authenticated:
# Navigation to and Selection of Advanced Search
browser.get(instrument_type_search_url)
# Ensure Page Elements Have Loaded
time.sleep(1)
# Reveal Additional Search Fields
search_type = browser.find_element_by_id("lbSearchType")
search_type.click()
# Ensure Page Elements Have Loaded
time.sleep(1)
# Issue an Advanced Search Query
start_date_field = browser.find_element_by_id("STARTDATE")
start_date_field.send_keys(start_date)
end_date_field = browser.find_element_by_id("ENDDATE")
end_date_field.send_keys(end_date)
instrument_type_field = browser.find_element_by_id("INSTRUMENT_TYPES")
instrument_type_field.send_keys(instrument_type)
search_button = browser.find_element_by_id("SearchButton")
search_button.click()
# Harvest the Query
document_links = set()
result_page_repeater_ids = set(
link.get_attribute("id")
for link in browser.find_elements_by_xpath("//a[@href]")
if "PageRepeater" in link.get_attribute("href"))
for result_page_repeater_id in result_page_repeater_ids:
result_page_repeater = browser.find_element_by_id(
result_page_repeater_id)
result_page_repeater.click()
current_page_document_links = set(
link.get_attribute("href")
for link in browser.find_elements_by_xpath("//a[@href]")
if "PK" in link.get_attribute("href"))
document_links = document_links | current_page_document_links
for result_page_link in document_links:
# Procedurally Extracting Instrument Types, Recording Dates,
# Dated Dates, Grantors, Grantees, & Property Addresses
malformed_property_address_counter = 0
browser.get(result_page_link)
column_data = browser.find_elements_by_class_name("coldata")
for column_datum in column_data:
parsed_datum = column_datum.text.split('\n')
if malformed_property_address_counter == 0:
instrument_types.append(parsed_datum[0])
elif malformed_property_address_counter == 4:
recording_dates.append(parsed_datum[0])
elif malformed_property_address_counter == 5:
dated_dates.append(parsed_datum[0])
elif malformed_property_address_counter == 9:
grantors.append(parsed_datum)
elif malformed_property_address_counter == 10:
grantees.append(parsed_datum)
malformed_property_address_counter += 1
view_document_button = browser.find_element_by_id(
"BTN_VIEW_DOCUMENT")
on_click = view_document_button.get_attribute("onclick")
pdf_download_link = base_search_url + str(on_click).split('\'')[1]
pdf = browser.request("GET", pdf_download_link).content
pdf_image = pdf2image.convert_from_bytes(pdf)
for page in pdf_image:
page_text = pytesseract.image_to_string(page)
if "Real Estate" in page_text \
or "PROPERTY TO BE DISTRIBUTED" in page_text \
or "Decedent resided at" in page_text \
or "Real Property" in page_text:
pdf_text = page_text
break
property_address = pyap.parse(pdf_text, country="US")
if len(property_address) > 0:
property_address = str(property_address[0])
result_page_links.append(result_page_link)
property_addresses.append(property_address)
update_activity_display(
"\n\n* Found " + str(len(property_addresses)) + " entries for \"" +
instrument_type + "\" cases in the Clay"
" County database. Compare"
" this number to the"
" number of populated fields"
" for property addresses in"
" the generated CSV file to"
" reconcile any"
" discrepancies that may"
" occur in the PDF"
" address-parsing process. *\n", activity_display)
valid_property_address_counter = 0
malformed_property_address_counter = 1
for result_page_link, property_address in zip(result_page_links,
property_addresses):
try:
if len(property_address) == 0:
raise NoSuchElementException
elif len(property_address) > 0:
# Navigation to Second Page
browser.get(gis_url)
# Ensure Page Elements Have Loaded
time.sleep(2)
# Click Agree
agree_field = browser.find_element_by_id(
"dojox_mobile_Button_0")
agree_field.click()
# Navigation to Address Search
address_search_field = browser.find_element_by_id(
"searchButton")
address_search_field.click()
# Ensure Page Elements Have Loaded
time.sleep(2)
address_search_tab = browser.find_element_by_id(
"addressTab")
address_search_tab.click()
# Ensure Page Elements Have Loaded
time.sleep(2)
search_input_field = browser.find_element_by_id(
"search_input")
# Ensure Page Elements Have Loaded
time.sleep(3)
# Enter Address
search_input_field.send_keys(property_address)
# Ensure Page Elements Have Loaded
time.sleep(2)
# Click Submit
search_input_field.send_keys(Keys.RETURN)
# Ensure Search Results Have Loaded
time.sleep(3)
# Harvesting Property Owner Names
gis_results_container = browser.find_element_by_id(
"resultsGeocodeContainer0")
for line in gis_results_container.text.split("\n"):
if "Current Owner" in line:
property_owner_name = line.split(":")[1].strip()
property_owner_names.append(property_owner_name)
break
# Collecting Property Owner Addresses
tabs = browser.window_handles
for link in gis_results_container.find_elements_by_xpath(
"//a[@href]"):
if "parcelid" in link.get_attribute("href"):
link.click()
tabs = browser.window_handles
browser.switch_to.window(tabs[1])
break
property_owner_addresses.append(
browser.find_element_by_xpath(
"/html/body/table[2]/tbody/tr[7]/td").text + ', ' +
browser.find_element_by_xpath(
"/html/body/table[2]/tbody/tr[9]/td").text)
browser.close()
browser.switch_to.window(tabs[0])
valid_property_address_counter += 1
else:
property_owner_names.append([])
property_owner_addresses.append([])
except NoSuchElementException:
malformed_property_address_result_page_link = result_page_link
update_activity_display(
"\nAn invalid address entered into the Clay County GIS address entry field was found.\n"
+ "It was located in the PDF found here:",
activity_display)
activity_display.config(state=NORMAL)
activity_display.pack()
activity_display.insert(
INSERT,
"Link to PDF #" + str(malformed_property_address_counter) +
" for Manual Inspection\n",
hyperlink.add(lambda: webbrowser.open(
malformed_property_address_result_page_link)))
activity_display.config(state=DISABLED)
property_owner_names.append([])
property_owner_addresses.append([])
malformed_property_address_counter += 1
pass
# Exporting Scraped Data to CSV File
df_data = [
instrument_types, recording_dates, dated_dates, grantors, grantees,
result_page_links, property_addresses, property_owner_names,
property_owner_addresses
]
df = pd.DataFrame(data=df_data).T
df.columns = [
"Instrument Type", "Recording Date", "Dated Date", "Grantor(s)",
"Grantee(s)", "Result Page Link", "Property Address",
"Current Property Owner Name", "Owner Mailing Address"
]
df.to_csv("Realtor Estate Data Export (" + str(date.today()) + ").csv")
# Cleaning Up the Browser Instance
browser.close()
return requests.get(instrument_type_search_url).status_code
class ObjSup:
def __init__(self):
print
self.start = time.time()
with open('settings.json', 'r') as settingsFile:
self.settings = json.loads(settingsFile.read())
self.headers = {
'Accept':
'*/*',
'Accept-Encoding':
'gzip, deflate, sdch',
'Accept-Language':
'en-US,en;q=0.8',
'Cache-Control':
'max-age=0',
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.81 Safari/537.36'
}
self.session = Chrome(
'/Users/alex/PycharmProjects/FirstSelenium/chromedriver')
self.session.headers = self.headers
self.found_link = ''
self.style_id = ''
self.prod_id = ''
self.form_key = ''
def cook(self):
self.log('Starting up')
self.log('Attempting to scrape category {}'.format(
self.settings['category']))
if self.scrape_product():
self.log('Attempting ATC')
if self.add_to_cart():
self.log('Attempting checkout')
if self.checkout():
self.log('Checked out')
else:
self.log('Failed to check out')
else:
self.log('Failed to ATC')
else:
self.log('Failed to scrape category')
def log(self, text):
current = datetime.now()
print('== {} == {}'.format(colored(str(current), 'blue'), str(text)))
def scrape_product(self):
cat_url = 'http://supremenewyork.com/shop/all/{}'.format(
self.settings['category'])
url_arr = []
name_arr = []
style_arr = []
self.log('Opening category page')
r = self.session.request('GET', cat_url, verify=False)
if r.status_code != 200:
self.log('Encountered bad status code {}'.format(r.status_code))
return False
soup = BeautifulSoup(r.text, 'html.parser')
for link in soup.select('a.name-link'):
url = link.get('href')
if url_arr.count(url) == 0:
name_arr.append(link.get_text())
url_arr.append(url)
else:
style_arr.append(link.get_text())
self.log('Found {} products ( {} names / {} styles)'.format(
str(len(url_arr)), str(len(name_arr)), str(len(style_arr))))
self.log('Checking against keyword "{}" & style "{}"'.format(
self.settings['name_key'], self.settings['style_key']))
for i in range(0, len(url_arr)):
if self.settings['name_key'] in name_arr[i].lower():
if self.settings['style_key'] in style_arr[i].lower():
self.found_link = url_arr[i]
self.log('Found matching link {}'.format(self.found_link))
return True
if not self.found_link:
self.log('Did not find a matching link')
return False
def add_to_cart(self):
url = 'http://supremenewyork.com/{}.json'.format(self.found_link)
self.log("Opening product JSON")
r = self.session.request('GET', url, verify=False)
if r.status_code != 200:
self.log('Encountered bad status {} opening product JSON'.format(
r.status_code))
return False
j = r.json()
self.log('Getting IDs')
for e in j['styles']:
if e['name'].lower() == self.settings['style_key']:
self.style_id = e['id']
self.log('Found style ID {}'.format(self.style_id))
for s in e['sizes']:
if s['name'].lower() == self.settings['size_key']:
if s['stock_level'] == 1:
self.prod_id = s['id']
self.log('Found product ID {}'.format(
self.prod_id))
if s['stock_level'] == 0:
self.log('Out of stock')
return False
# The form key entry point seems to be not working at the moment
# self.log('Looking for form key')
# url = 'http://supremenewyork.com/{}'.format(self.found_link)
# r = self.session.request("GET", url, verify=False)
# if r.status_code != 200:
# self.log('Bad status code {} when looking for form key'.format(r.status_code))
# return False
# soup = BeautifulSoup(r.text, 'html.parser')
# sources = soup.findAll('form', {"action": True})
# for s in sources:
# self.form_key = s['action']
# self.log('Found form key {}'.format(self.form_key))
# time.sleep(2)
# url = 'http://www.supremenewyork.com/{}'.format(self.form_key)
# p = "commit=add%20to%20cart&style={}&size={}&utf8=%E2%9C%93".format(self.style_id, self.prod_id)
# r = self.session.request('POST', url, data=p, verify=False)
# if r.status_code != 200:
# self.log('Bad status code {} when firing form POST'.format(r.status_code))
# return False
self.session.get('http://www.supremenewyork.com/{}'.format(
self.found_link))
s = Select(self.session.find_element_by_xpath("//select[@id='size']"))
s.select_by_value(str(self.prod_id))
button = self.session.find_element_by_name("commit")
button.click()
time.sleep(1)
self.log('Checking cart contents')
r = self.session.request(
'GET',
'http://www.supremenewyork.com/shop/cart.json',
verify=False)
if str(self.prod_id) not in r.text:
self.log('Product ID not in cart contents')
return False
return True
def checkout(self):
# This is a collection of the form elements. The selectors might change
# so I try and adapt by using tab keys.
self.session.get('https://www.supremenewyork.com/checkout')
self.log('Finding form elements')
name = self.session.find_element_by_id('order_billing_name')
# email = self.session.find_element_by_id('order_email')
tel = self.session.find_element_by_id('order_tel')
# add = self.session.find_element_by_name('order[billing_address]')
# ad2 = self.session.find_element_by_name('order[billing_address_2]')
# zip = self.session.find_element_by_id('order_billing_zip')
# city = self.session.find_element_by_id('order_billing_city')
# state = Select(self.session.find_element_by_id('order_billing_state'))
# country = Select(self.session.find_element_by_id('order_billing_country'))
ctype = Select(self.session.find_element_by_id('credit_card_type'))
# cc = self.session.find_element_by_name('order[cnb]')
xm = Select(self.session.find_element_by_id('credit_card_month'))
xy = Select(self.session.find_element_by_id('credit_card_year'))
# cvv = self.session.find_element_by_xpath("//input[@size='4']")
submit = self.session.find_element_by_xpath("//input[@type='submit']")
self.log('Entering checkout details')
name.send_keys(self.settings['f_name'] + ' ' + self.settings['l_name'],
Keys.TAB, self.settings['email'], Keys.TAB)
tel.send_keys(self.settings['phone'], Keys.TAB,
self.settings['address'], Keys.TAB,
self.settings['address2'], Keys.TAB,
self.settings['zip'], Keys.TAB)
ctype.select_by_visible_text(self.settings['type'])
cardchain = ActionChains(self.session)
cardchain.send_keys(Keys.TAB, self.settings['cc'])
cardchain.perform()
xm.select_by_value(self.settings['month'])
xy.select_by_value(self.settings['year'])
cvvchain = ActionChains(self.session)
cvvchain.send_keys(Keys.TAB, self.settings['cvv'], Keys.TAB,
Keys.SPACE)
cvvchain.perform()
# This is the delay you want to set to avoid ghost checkout
time.sleep(self.settings['delay'])
submit.click()
return True
def login(user_email_address, user_password, is_headless, two_factor_token):
"""
Attempts to log into Facebook
Returns a driver object
user_email_address: str Your Email
user_password: str Your password
user_profile_url: str Your profile URL
"""
# The Chrome driver is required because Gecko was having issues
chrome_options = Options()
prefs = {
"profile.default_content_setting_values.notifications": 2,
'disk-cache-size': 4096
}
chrome_options.add_experimental_option("prefs", prefs)
chrome_options.add_argument("start-maximized")
if is_headless:
chrome_options.add_argument('--headless')
chrome_options.add_argument('--disable-gpu')
chrome_options.add_argument('log-level=2')
try:
driver = Chrome(options=chrome_options)
except Exception as e:
# The user does not have chromedriver installed
# Tell them to install it
stderr.write(str(e))
stderr.write(no_chrome_driver)
exit(1)
driver.implicitly_wait(10)
driver.get("https://facebook.com")
email = "email"
password = "******"
login = "******"
approvals_code = "approvals_code"
emailelement = driver.find_element_by_name(email)
passwordelement = driver.find_element_by_name(password)
emailelement.send_keys(user_email_address)
passwordelement.send_keys(user_password)
loginelement = driver.find_element_by_id(login)
loginelement.click()
# Defaults to no 2fa
has_2fa = False
try:
# If this element exists, we've reached a 2FA page
driver.find_element_by_xpath("//form[@class=\"checkpoint\"]")
driver.find_element_by_xpath("//input[@name=\"approvals_code\"]")
has_2fa = True
except NoSuchElementException:
has_2fa = "two-factor authentication" in driver.page_source.lower(
) or has_2fa
if has_2fa:
print("""
Two-Factor Auth is enabled.
Please file an issue at https://github.com/weskerfoot/DeleteFB/issues if you run into any problems
""")
if two_factor_token and has_2fa:
twofactorelement = driver.find_element_by_name(approvals_code)
twofactorelement.send_keys(two_factor_token)
# Submits after the code is passed into the form, does not validate 2FA code.
contelement = driver.find_element_by_id("checkpointSubmitButton")
contelement.click()
# Defaults to saving this new browser, this occurs on each new automated login.
save_browser = driver.find_element_by_id("checkpointSubmitButton")
save_browser.click()
elif has_2fa:
# Allow time to enter 2FA code
print("Pausing to enter 2FA code")
time.sleep(35)
print("Continuing execution")
else:
pass
return driver
class Task:
#takes arguments and creates class attributes for easier and prettier access
def __init__(self, name, incart, checkout, settings):
self.cart = [] #array of item objects to cart and checkout
self.settings = settings #dictionairy of user settings
for item in incart:
self.cart.append(
Item(item, self.settings
)) #convert array of arrays to array of item objects
self.checkout = checkout #dictionairy of checkout values (such as name, address, ect)
self.name = name #name of task to be used during printing to console
#creates a selenium webdriver
def create_webdriver(self):
#https://stackoverflow.com/questions/29916054/change-user-agent-for-selenium-driver
opts = Options()
self.useragent = random_user_agent() #calls for a random useragent
opts.add_argument("user-agent=" +
self.useragent) #adds random user agent to options
self.webdriver = Chrome(
executable_path=get_local_directory() + '/resources/chromedriver',
chrome_options=opts) #creates new webdriver with premade options
self.webdriver.set_page_load_timeout(int(
self.settings["timeout"])) #set timeout of pageload from config
#attempts to cart every item in cart. Returns true if any of the items cart
def cart_items(self):
self.create_webdriver()
return_bool = False
for item in self.cart:
if (item.in_cart == False):
item.cart(self.webdriver)
if (item.in_cart == True):
return_bool = True
return return_bool
#loads checkout page, fills form, clicks checkout, waits for user to complete captcha
def manual_checkout(self):
self.webdriver.get(
'https://www.supremenewyork.com/checkout') #load the checkout page
self.fill_form()
self.webdriver.find_element_by_name("commit").click()
if (self.wait_for_manual_captcha()):
return self.confirmation()
#fills all elements of form and check required check boxes
def fill_form(self):
#TODO: IF SITEKEY CHANGED, CLEAR OLD TOKENS (not high priority)
update_sitekey(
self.webdriver.find_element_by_xpath(
'//*[@id="cart-cc"]/fieldset/div[3]').get_attribute(
"data-sitekey")
) #updates the captcha sitekey saved on file (in case it changed).
self.webdriver.find_element_by_name("order[billing_name]").send_keys(
self.checkout["name"])
self.webdriver.find_element_by_name("order[email]").send_keys(
self.checkout["email"])
self.webdriver.find_element_by_name("order[tel]").send_keys(
self.checkout["phone"])
self.webdriver.find_element_by_name(
"order[billing_address]").send_keys(self.checkout["address"])
self.webdriver.find_element_by_name("order[billing_zip]").send_keys(
self.checkout["zipcode"])
self.webdriver.find_element_by_name("order[billing_city]").send_keys(
self.checkout["city"])
self.webdriver.find_element_by_name("order[billing_state]").send_keys(
self.checkout["state"])
self.webdriver.find_element_by_name(
"order[billing_country]").send_keys(self.checkout["country"])
checkout_field = self.webdriver.find_element_by_name(
"credit_card[nlb]")
for character in list(
self.checkout["card_number"]
): #weird error occurs when you send all at once,was shuffling characters
time.sleep(.01)
checkout_field.send_keys(character)
self.webdriver.find_element_by_name("credit_card[month]").send_keys(
self.checkout["card_month"])
self.webdriver.find_element_by_name("credit_card[year]").send_keys(
self.checkout["card_year"])
self.webdriver.find_element_by_name("credit_card[rvv]").send_keys(
self.checkout["cvv"])
self.webdriver.find_element_by_xpath(
"//*[@id='cart-cc']/fieldset/p[2]/label/div/ins").click()
#loads checkout page, fills form(supreme supplies necessary cookies during form filling, so even though I never submit I need to fill the form)
def ajax_checkout(self):
self.webdriver.get('https://www.supremenewyork.com/checkout')
if ("y" in self.settings["fill_form"]):
try:
self.fill_form()
except NoSuchElementException:
pass
csrf_token = self.webdriver.find_element_by_name(
'csrf-token').get_attribute("content")
#headers required to make AJAX request, found using chrome devtools
headers = {
'Accept': '*/*',
'X-CSRF-Token': csrf_token,
'X-Requested-With': 'XMLHttpRequest',
'Referer': 'https://www.supremenewyork.com/checkout',
'Accept-Language': 'en-US,en;q=0.8',
'User-Agent': self.useragent,
'Connection': 'keep-alive',
#'Host':'wwww.supremenewyork.com',
'Origin': 'https://www.supremenewyork.com',
'Accept-Encoding': 'gzip, deflate, br',
'Content-Length': '1006',
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8'
}
#payload required to make AJAX request, found using chrome devtools
payload = {
'utf8': '✓',
'authenticity_token': csrf_token,
'order[billing_name]': self.checkout['name'],
'order[email]': self.checkout['email'],
'order[tel]': self.checkout['phone'],
'order[billing_address]': self.checkout['address'],
'order[billing_address_2]': '',
'order[billing_zip]': self.checkout['zipcode'],
'order[billing_city]': self.checkout['city'],
'order[billing_state]': self.checkout['state'],
'order[billing_country]': self.checkout['country'],
'same_as_billing_address': '1',
'asec': 'Rmasn',
'store_credit_id': '',
'credit_card[nlb]': self.checkout["card_number"],
'credit_card[month]': self.checkout["card_month"],
'credit_card[year]': self.checkout["card_year"],
'credit_card[rvv]': self.checkout["cvv"],
'order[terms]': '0',
'order[terms]': '1',
'credit_card[vval]': self.checkout["cvv"],
'g-recaptcha-response': get_captcha()
}
#make request, print response
response = self.webdriver.request(
'POST',
'https://www.supremenewyork.com/checkout.json',
data=payload,
headers=headers)
print "[" + self.name + "] RESPONSE: " + response.text
#looks to see if captchas solved yet
def wait_for_manual_captcha(self):
while (True):
#trys to click checkout, if captcha is visible it will throw WebDriverException
try:
self.webdriver.find_element_by_name("commit").click()
return True
except WebDriverException:
print "[" + self.name + "] Fill captcha manually..."
time.sleep(3)
#finds result of checkout on confirmation page
def confirmation(self):
while True:
try:
if ('selected' in self.webdriver.find_element_by_xpath(
'//*[@id="tabs"]/div[3]').get_attribute('class')
): #if the confirmation tab is selected
print "[" + self.name + "] Response: " + self.webdriver.find_element_by_id(
'content'
).text.split("CONFIRMATION")[
1] #print the desired information from checkout page
return True
except (NoSuchElementException, StaleElementReferenceException
): #thrown if tab is not selected
print "[" + self.name + "] Waiting for confirmation..."
time.sleep(1)
def get_web_driver(email, password, headless=False, mfa_method=None,
mfa_input_callback=None, wait_for_sync=True):
if headless and mfa_method is None:
warnings.warn("Using headless mode without specifying an MFA method"
"is unlikely to lead to a successful login. Defaulting --mfa-method=sms")
mfa_method = "sms"
zip_type = ""
executable_path = os.getcwd()
if _platform == "linux" or _platform == "linux2":
zip_type = 'linux'
executable_path += "/chromedriver"
elif _platform == "darwin":
zip_type = 'mac'
executable_path += "/chromedriver"
elif _platform == "win32" or _platform == "win64":
zip_type = 'win'
executable_path += "/chromedriver.exe"
if not os.path.exists(executable_path):
zip_file_url = 'https://chromedriver.storage.googleapis.com/%s/chromedriver_%s64.zip' % (CHROME_DRIVER_VERSION,
zip_type)
request = requests.get(zip_file_url)
zip_file = zipfile.ZipFile(io.BytesIO(request.content))
zip_file.extractall()
os.chmod(executable_path, 0o755)
chrome_options = ChromeOptions()
if headless:
chrome_options.add_argument('headless')
chrome_options.add_argument('no-sandbox')
chrome_options.add_argument('disable-dev-shm-usage')
chrome_options.add_argument('disable-gpu')
# chrome_options.add_argument("--window-size=1920x1080")
driver = Chrome(chrome_options=chrome_options, executable_path="%s" % executable_path)
driver.get("https://www.mint.com")
driver.implicitly_wait(20) # seconds
driver.find_element_by_link_text("Log In").click()
driver.find_element_by_id("ius-userid").send_keys(email)
driver.find_element_by_id("ius-password").send_keys(password)
driver.find_element_by_id("ius-sign-in-submit-btn").submit()
# Wait until logged in, just in case we need to deal with MFA.
while not driver.current_url.startswith(
'https://mint.intuit.com/overview.event'):
time.sleep(1)
driver.implicitly_wait(1) # seconds
try:
driver.find_element_by_id('ius-mfa-options-form')
try:
mfa_method_option = driver.find_element_by_id('ius-mfa-option-{}'.format(mfa_method))
mfa_method_option.click()
mfa_method_submit = driver.find_element_by_id("ius-mfa-options-submit-btn")
mfa_method_submit.click()
mfa_code = (mfa_input_callback or input)("Please enter your 6-digit MFA code: ")
mfa_code_input = driver.find_element_by_id("ius-mfa-confirm-code")
mfa_code_input.send_keys(mfa_code)
mfa_code_submit = driver.find_element_by_id("ius-mfa-otp-submit-btn")
mfa_code_submit.click()
except Exception: # if anything goes wrong for any reason, give up on MFA
mfa_method = None
warnings.warn("Giving up on handling MFA. Please complete "
"the MFA process manually in the browser.")
except NoSuchElementException:
pass
finally:
driver.implicitly_wait(20) # seconds
# Wait until the overview page has actually loaded, and if wait_for_sync==True, sync has completed.
if wait_for_sync:
status_message = driver.find_element_by_css_selector(".SummaryView .message")
try:
WebDriverWait(driver, 5 * 60).until(
lambda x: status_message.get_attribute('innerHTML') == "Account refresh completed."
)
except TimeoutException:
warnings.warn("Mint sync apparently incomplete after 5 minutes. Data "
"retrieved may not be current.")
else:
driver.find_element_by_id("transaction")
return driver
def get_amzn_driver(email, password, headless=False, session_path=None):
zip_type = ""
executable_path = os.path.join(os.getcwd(), 'chromedriver')
if _platform in ['win32', 'win64']:
executable_path += '.exe'
zip_type = CHROME_ZIP_TYPES.get(_platform)
if not os.path.exists(executable_path):
zip_file_url = CHROME_DRIVER_BASE_URL.format(CHROME_DRIVER_VERSION,
zip_type)
request = requests.get(zip_file_url)
if request.status_code != 200:
raise RuntimeError(
'Error finding chromedriver at {}, status = {}'.format(
zip_file_url, request.status_code))
zip_file = zipfile.ZipFile(io.BytesIO(request.content))
zip_file.extractall()
os.chmod(executable_path, 0o755)
chrome_options = ChromeOptions()
if headless:
chrome_options.add_argument('headless')
chrome_options.add_argument('no-sandbox')
chrome_options.add_argument('disable-dev-shm-usage')
chrome_options.add_argument('disable-gpu')
# chrome_options.add_argument("--window-size=1920x1080")
if session_path is not None:
chrome_options.add_argument("user-data-dir=" + session_path)
logger.info('Logging into Amazon.com')
driver = Chrome(chrome_options=chrome_options,
executable_path=executable_path)
driver.get(ORDER_HISTORY_URL_VIA_SWITCH_ACCOUNT_LOGIN)
driver.implicitly_wait(2)
def get_element_by_id(driver, id):
try:
return driver.find_element_by_id(id)
except NoSuchElementException:
pass
return None
def get_element_by_xpath(driver, xpath):
try:
return driver.find_element_by_xpath(xpath)
except NoSuchElementException:
pass
return None
# Go straight to the account switcher, and look for the given email.
# If present, click on it! Otherwise, click on "Add account".
desired_account_element = get_element_by_xpath(
driver, "//div[contains(text(), '{}')]".format(email))
if desired_account_element:
desired_account_element.click()
driver.implicitly_wait(2)
# It's possible this account has already authed recently. If so, the
# next block will be skipped and the login is complete!
if not get_element_by_id(driver, 'report-confirm'):
driver.find_element_by_id('ap_password').send_keys(
get_password(password))
driver.find_element_by_name('rememberMe').click()
driver.find_element_by_id('signInSubmit').submit()
else:
# Cannot find the desired account in the switch. Log in via Add Account
driver.find_element_by_xpath('//div[text()="Add account"]').click()
driver.implicitly_wait(2)
driver.find_element_by_id('ap_email').send_keys(email)
driver.find_element_by_id('ap_password').send_keys(
get_password(password))
driver.find_element_by_name('rememberMe').click()
driver.find_element_by_id('signInSubmit').submit()
driver.implicitly_wait(2)
if not get_element_by_id(driver, 'report-confirm'):
logger.warning('Having trouble logging into Amazon. Please see the '
'browser and complete login within the next 5 minutes. '
'This script will continue automatically on success. '
'You may need to manually navigate to: {}'.format(
ORDER_HISTORY_REPORT_URL))
if get_element_by_id(driver, 'auth-mfa-otpcode'):
logger.warning('Hint: Looks like an auth challenge! Maybe check '
'your email')
try:
wait_cond = EC.presence_of_element_located((By.ID, 'report-confirm'))
WebDriverWait(driver, 60 * 5).until(wait_cond)
except TimeoutException:
logger.critical('Cannot complete login!')
exit(1)
return driver
def get_web_driver(email, password, headless=False, mfa_method=None,
mfa_input_callback=None, wait_for_sync=True,
session_path=None, imap_account=None, imap_password=None,
imap_server=None, imap_folder="INBOX"):
if headless and mfa_method is None:
warnings.warn("Using headless mode without specifying an MFA method"
"is unlikely to lead to a successful login. Defaulting --mfa-method=sms")
mfa_method = "sms"
zip_type = ""
executable_path = os.getcwd() + os.path.sep + 'chromedriver'
if _platform in ['win32', 'win64']:
executable_path += '.exe'
zip_type = CHROME_ZIP_TYPES.get(_platform)
if not os.path.exists(executable_path):
zip_file_url = CHROME_DRIVER_BASE_URL % (CHROME_DRIVER_VERSION, zip_type)
request = requests.get(zip_file_url)
if request.status_code != 200:
raise RuntimeError('Error finding chromedriver at %r, status = %d' %
(zip_file_url, request.status_code))
zip_file = zipfile.ZipFile(io.BytesIO(request.content))
zip_file.extractall()
os.chmod(executable_path, 0o755)
chrome_options = ChromeOptions()
if headless:
chrome_options.add_argument('headless')
chrome_options.add_argument('no-sandbox')
chrome_options.add_argument('disable-dev-shm-usage')
chrome_options.add_argument('disable-gpu')
# chrome_options.add_argument("--window-size=1920x1080")
if session_path is not None:
chrome_options.add_argument("user-data-dir=%s" % session_path)
driver = Chrome(chrome_options=chrome_options, executable_path="%s" % executable_path)
driver.get("https://www.mint.com")
driver.implicitly_wait(20) # seconds
try:
element = driver.find_element_by_link_text("Log In")
except NoSuchElementException:
# when user has cookies, a slightly different front page appears
driver.implicitly_wait(0) # seconds
element = driver.find_element_by_link_text("LOG IN")
driver.implicitly_wait(20) # seconds
element.click()
time.sleep(1)
email_input = driver.find_element_by_id("ius-userid")
# It's possible that the user clicked "remember me" at some point, causing
# the email to already be present. If anything is in the input, clear it
# and use the provided email, just to be safe.
# email_input.setAttribute("value", "")
email_input.clear()
email_input.send_keys(email)
driver.find_element_by_id("ius-password").send_keys(password)
driver.find_element_by_id("ius-sign-in-submit-btn").submit()
# Wait until logged in, just in case we need to deal with MFA.
while not driver.current_url.startswith(
'https://mint.intuit.com/overview.event'):
# An implicitly_wait is also necessary here to avoid getting stuck on
# find_element_by_id while the page is still in transition.
driver.implicitly_wait(1)
time.sleep(1)
# bypass "Let's add your current mobile number" interstitial page
try:
skip_for_now = driver.find_element_by_id('ius-verified-user-update-btn-skip')
skip_for_now.click()
except (NoSuchElementException, StaleElementReferenceException, ElementNotVisibleException):
pass
driver.implicitly_wait(1) # seconds
try:
driver.find_element_by_id('ius-mfa-options-form')
try:
mfa_method_option = driver.find_element_by_id('ius-mfa-option-{}'.format(mfa_method))
mfa_method_option.click()
mfa_method_submit = driver.find_element_by_id("ius-mfa-options-submit-btn")
mfa_method_submit.click()
if mfa_method == 'email' and imap_account:
mfa_code = get_email_code(imap_account, imap_password, imap_server, imap_folder=imap_folder)
else:
mfa_code = (mfa_input_callback or input)("Please enter your 6-digit MFA code: ")
mfa_code_input = driver.find_element_by_id("ius-mfa-confirm-code")
mfa_code_input.send_keys(mfa_code)
mfa_code_submit = driver.find_element_by_id("ius-mfa-otp-submit-btn")
mfa_code_submit.click()
except Exception: # if anything goes wrong for any reason, give up on MFA
mfa_method = None
warnings.warn("Giving up on handling MFA. Please complete "
"the MFA process manually in the browser.")
except NoSuchElementException:
pass
finally:
driver.implicitly_wait(20) # seconds
# Wait until the overview page has actually loaded, and if wait_for_sync==True, sync has completed.
if wait_for_sync:
try:
# Status message might not be present straight away. Seems to be due
# to dynamic content (client side rendering).
status_message = WebDriverWait(driver, 30).until(
expected_conditions.visibility_of_element_located(
(By.CSS_SELECTOR, ".SummaryView .message")))
WebDriverWait(driver, 5 * 60).until(
lambda x: "Account refresh complete" in status_message.get_attribute('innerHTML')
)
except (TimeoutException, StaleElementReferenceException):
warnings.warn("Mint sync apparently incomplete after 5 minutes. Data "
"retrieved may not be current.")
else:
driver.find_element_by_id("transaction")
return driver
def send_outlook():
home_dir = str(Path.home())
chrome_cache_path = f"{home_dir}/.chrome_cache"
print(f"loading chrome, caching to: {chrome_cache_path}")
chrome_options = Options()
chrome_options.add_argument("--headless")
chrome_options.add_argument("--no-startup-window")
chrome_options.add_argument("--disable-gpu")
chrome_options.add_argument("--disable-sync-preferences")
chrome_options.add_argument("--disable-extensions")
chrome_options.add_argument("--disable-background-networking")
chrome_options.add_argument("--no-first-run")
chrome_options.add_argument("--aggressive-tab-discard")
chrome_options.add_argument("--user-agent=Mozilla/4.0 (Windows; MSIE 6.0; Windows NT 5.2)")
chrome_options.add_argument(f"--user-data-dir={chrome_cache_path}/user-data")
chrome_options.add_argument(f"--data-path={chrome_cache_path}/data-path")
chrome_options.add_argument(f"--disk-cache-dir={chrome_cache_path}/disk-cache")
chrome_options.add_argument(f"--homedir={chrome_cache_path}")
chrome_options.add_argument(f"--disk-cache-dir={chrome_cache_path}/cache-dir")
prefs={"profile.managed_default_content_settings.images": 2, 'disk-cache-size': 4096 }
chrome_options.add_experimental_option("prefs",prefs)
delay = 60
chrome_options.binary_location = "/usr/bin/chromium-browser"
driver = Chrome(executable_path=os.path.abspath("/usr/lib/chromium-browser/chromedriver"), chrome_options=chrome_options)
print("logging into outlook")
driver.get("https://outlook.office.com/owa/")
try:
driver.find_element_by_name("loginfmt").send_keys("*****@*****.**")
driver.find_element_by_id("idSIButton9").click()
print("entered username, waiting for password prompt")
try:
myElem = WebDriverWait(driver, delay).until(EC.presence_of_element_located((By.ID, 'passwordInput')))
print("password prompt loaded")
except TimeoutException:
print("Loading password prompt took too much time!")
print(driver.page_source)
driver.close();
exit(1)
passwd = getpass.getpass()
driver.find_element_by_id("passwordInput").send_keys(passwd)
driver.find_element_by_id("submitButton").click()
print("entered password, waiting for 2FA token")
try:
myElem = WebDriverWait(driver, delay).until(EC.presence_of_element_located((By.ID, 'idSIButton9')))
driver.find_element_by_id("idSIButton9").click()
print("asking to remember credentials for next time")
except TimeoutException:
print("Loading 2FA page took too much time!")
print(driver.page_source)
driver.close();
exit(1)
print("2FA accepted, loading office landing page")
except NoSuchElementException:
print("already logged in")
try:
print("waiting for landing page to load")
myElem = WebDriverWait(driver, delay).until(EC.presence_of_element_located((By.ID, 'lnkBrwsAllFldrs')))
except TimeoutException:
print(driver.page_source)
print("Loading landing page too much time!")
driver.close();
exit(1)
try:
eink.send_update("Loading Tasks")
print("loading tasks")
driver.find_element_by_id("lnkBrwsAllFldrs").click()
driver.find_element_by_id("selbrfld").click()
Select(driver.find_element_by_id("selbrfld")).select_by_visible_text("Tasks")
driver.find_element_by_id("selbrfld").click()
driver.find_element_by_xpath("(.//*[normalize-space(text()) and normalize-space(.)='Sent Items'])[1]/following::img[1]").click()
myElem = WebDriverWait(driver, delay).until(EC.presence_of_element_located((By.CLASS_NAME, 'lvw')))
except TimeoutException:
print(driver.page_source)
print("Loading todo list took too much time!")
driver.close();
exit(1)
elements = driver.find_elements_by_css_selector("td h1 a")
for i, element in enumerate(elements):
eink.send_todo(i, element.text)
try:
eink.send_update("Loading Calendar")
print("loading calendar")
driver.find_element_by_id("lnkNavCal").click()
myElem = WebDriverWait(driver, delay).until(EC.presence_of_element_located((By.CLASS_NAME, 'cntnttp')))
print("calendar loaded, dumping entries")
except TimeoutException:
print(driver.page_source)
print("Loading calendar took too much time!")
driver.close();
exit(1)
elements = driver.find_elements_by_css_selector("td.v a")
for i, element in enumerate(elements):
eink.send_meeting(i,element.get_attribute('title'))