def delete_posts(user_email_address=None,
user_password=None,
user_profile_url=None):
"""
user_email_address: Your Email
user_password: Your password
user_profile_url: Your profile URL
"""
assert all((user_email_address,
user_password,
user_profile_url)), "Insufficient arguments provided"
chrome_options = Options()
prefs = {"profile.default_content_setting_values.notifications" : 2}
chrome_options.add_experimental_option("prefs", prefs)
chrome_options.add_argument("start-maximized")
driver = Chrome(chrome_options=chrome_options)
driver.implicitly_wait(10)
driver.get("https://facebook.com")
email = "email"
password = "******"
login="******"
emailelement = driver.find_element_by_name(email)
passwordelement = driver.find_element_by_name(password)
emailelement.send_keys(user_email_address)
passwordelement.send_keys(user_password)
loginelement = driver.find_element_by_id(login)
loginelement.click()
driver.get(user_profile_url)
for _ in range(MAX_POSTS):
post_button_sel = "_4xev"
timeline_element = driver.find_element_by_class_name(post_button_sel)
actions = ActionChains(driver)
actions.move_to_element(timeline_element).click().perform()
menu = driver.find_element_by_css_selector("#globalContainer > div.uiContextualLayerPositioner.uiLayer > div")
actions.move_to_element(menu).perform()
try:
delete_button = menu.find_element_by_xpath("//a[@data-feed-option-name=\"FeedDeleteOption\"]")
except:
delete_button = menu.find_element_by_xpath("//a[@data-feed-option-name=\"HIDE_FROM_TIMELINE\"]")
actions.move_to_element(delete_button).click().perform()
confirmation_button = driver.find_element_by_class_name("layerConfirm")
driver.execute_script("arguments[0].click();", confirmation_button)
time.sleep(5)
driver.refresh()
def delete_posts(user_email_address, user_password, user_profile_url,
is_headless):
"""
user_email_address: str Your Email
user_password: str Your password
user_profile_url: str Your profile URL
"""
# The Chrome driver is required because Gecko was having issues
chrome_options = Options()
prefs = {
"profile.default_content_setting_values.notifications": 2,
'disk-cache-size': 4096
}
chrome_options.add_experimental_option("prefs", prefs)
chrome_options.add_argument("start-maximized")
if is_headless:
chrome_options.add_argument('--headless')
chrome_options.add_argument('--disable-gpu')
chrome_options.add_argument('log-level=2')
driver = Chrome(options=chrome_options)
driver.implicitly_wait(10)
driver.get("https://facebook.com")
email = "email"
password = "******"
login = "******"
emailelement = driver.find_element_by_name(email)
passwordelement = driver.find_element_by_name(password)
emailelement.send_keys(user_email_address)
passwordelement.send_keys(user_password)
loginelement = driver.find_element_by_id(login)
loginelement.click()
if "Two-factor authentication" in driver.page_source:
# Allow time to enter 2FA code
print("Pausing to enter 2FA code")
time.sleep(20)
print("Continuing execution")
driver.get(user_profile_url)
for _ in range(MAX_POSTS):
post_button_sel = "_4xev"
while True:
try:
timeline_element = driver.find_element_by_class_name(
post_button_sel)
actions = ActionChains(driver)
actions.move_to_element(timeline_element).click().perform()
menu = driver.find_element_by_css_selector(
"#globalContainer > div.uiContextualLayerPositioner.uiLayer > div"
)
actions.move_to_element(menu).perform()
try:
delete_button = menu.find_element_by_xpath(
"//a[@data-feed-option-name=\"FeedDeleteOption\"]")
except SELENIUM_EXCEPTIONS:
delete_button = menu.find_element_by_xpath(
"//a[@data-feed-option-name=\"HIDE_FROM_TIMELINE\"]")
actions.move_to_element(delete_button).click().perform()
confirmation_button = driver.find_element_by_class_name(
"layerConfirm")
# Facebook would not let me get focus on this button without some custom JS
driver.execute_script("arguments[0].click();",
confirmation_button)
except SELENIUM_EXCEPTIONS:
continue
else:
break
# Required to sleep the thread for a bit after using JS to click this button
time.sleep(5)
driver.refresh()
def get_web_driver(email, password, headless=False, mfa_method=None,
mfa_input_callback=None, wait_for_sync=True):
if headless and mfa_method is None:
warnings.warn("Using headless mode without specifying an MFA method"
"is unlikely to lead to a successful login. Defaulting --mfa-method=sms")
mfa_method = "sms"
zip_type = ""
executable_path = os.getcwd()
if _platform == "linux" or _platform == "linux2":
zip_type = 'linux'
executable_path += "/chromedriver"
elif _platform == "darwin":
zip_type = 'mac'
executable_path += "/chromedriver"
elif _platform == "win32" or _platform == "win64":
zip_type = 'win'
executable_path += "/chromedriver.exe"
if not os.path.exists(executable_path):
zip_file_url = 'https://chromedriver.storage.googleapis.com/%s/chromedriver_%s64.zip' % (CHROME_DRIVER_VERSION,
zip_type)
request = requests.get(zip_file_url)
zip_file = zipfile.ZipFile(io.BytesIO(request.content))
zip_file.extractall()
os.chmod(executable_path, 0o755)
chrome_options = ChromeOptions()
if headless:
chrome_options.add_argument('headless')
chrome_options.add_argument('no-sandbox')
chrome_options.add_argument('disable-dev-shm-usage')
chrome_options.add_argument('disable-gpu')
# chrome_options.add_argument("--window-size=1920x1080")
driver = Chrome(chrome_options=chrome_options, executable_path="%s" % executable_path)
driver.get("https://www.mint.com")
driver.implicitly_wait(20) # seconds
driver.find_element_by_link_text("Log In").click()
driver.find_element_by_id("ius-userid").send_keys(email)
driver.find_element_by_id("ius-password").send_keys(password)
driver.find_element_by_id("ius-sign-in-submit-btn").submit()
# Wait until logged in, just in case we need to deal with MFA.
while not driver.current_url.startswith(
'https://mint.intuit.com/overview.event'):
time.sleep(1)
driver.implicitly_wait(1) # seconds
try:
driver.find_element_by_id('ius-mfa-options-form')
try:
mfa_method_option = driver.find_element_by_id('ius-mfa-option-{}'.format(mfa_method))
mfa_method_option.click()
mfa_method_submit = driver.find_element_by_id("ius-mfa-options-submit-btn")
mfa_method_submit.click()
mfa_code = (mfa_input_callback or input)("Please enter your 6-digit MFA code: ")
mfa_code_input = driver.find_element_by_id("ius-mfa-confirm-code")
mfa_code_input.send_keys(mfa_code)
mfa_code_submit = driver.find_element_by_id("ius-mfa-otp-submit-btn")
mfa_code_submit.click()
except Exception: # if anything goes wrong for any reason, give up on MFA
mfa_method = None
warnings.warn("Giving up on handling MFA. Please complete "
"the MFA process manually in the browser.")
except NoSuchElementException:
pass
finally:
driver.implicitly_wait(20) # seconds
# Wait until the overview page has actually loaded, and if wait_for_sync==True, sync has completed.
if wait_for_sync:
status_message = driver.find_element_by_css_selector(".SummaryView .message")
try:
WebDriverWait(driver, 5 * 60).until(
lambda x: status_message.get_attribute('innerHTML') == "Account refresh completed."
)
except TimeoutException:
warnings.warn("Mint sync apparently incomplete after 5 minutes. Data "
"retrieved may not be current.")
else:
driver.find_element_by_id("transaction")
return driver
def delete_posts(user_email_address=None,
user_password=None,
user_profile_url=None):
"""
user_email_address: Your Email
user_password: Your password
user_profile_url: Your profile URL
"""
assert all((user_email_address, user_password,
user_profile_url)), "Insufficient arguments provided"
# The Chrome driver is required because Gecko was having issues
chrome_options = Options()
prefs = {"profile.default_content_setting_values.notifications": 2}
chrome_options.add_experimental_option("prefs", prefs)
chrome_options.add_argument("start-maximized")
driver = Chrome(chrome_options=chrome_options)
driver.implicitly_wait(10)
driver.get("https://facebook.com")
email = "email"
password = "******"
login = "******"
emailelement = driver.find_element_by_name(email)
passwordelement = driver.find_element_by_name(password)
emailelement.send_keys(user_email_address)
passwordelement.send_keys(user_password)
loginelement = driver.find_element_by_id(login)
loginelement.click()
if "Two-factor authentication" in driver.page_source:
# Allow time to enter 2FA code
print("Pausing to enter 2FA code")
time.sleep(20)
print("Continuing execution")
driver.get(user_profile_url)
for _ in range(MAX_POSTS):
post_button_sel = "_4xev"
timeline_element = driver.find_element_by_class_name(post_button_sel)
actions = ActionChains(driver)
actions.move_to_element(timeline_element).click().perform()
menu = driver.find_element_by_css_selector(
"#globalContainer > div.uiContextualLayerPositioner.uiLayer > div")
actions.move_to_element(menu).perform()
try:
delete_button = menu.find_element_by_xpath(
"//a[@data-feed-option-name=\"FeedDeleteOption\"]")
# FIXME Using a bare except here to avoid having to handle all possible exceptions
except:
delete_button = menu.find_element_by_xpath(
"//a[@data-feed-option-name=\"HIDE_FROM_TIMELINE\"]")
actions.move_to_element(delete_button).click().perform()
confirmation_button = driver.find_element_by_class_name("layerConfirm")
# Facebook would not let me get focus on this button without some custom JS
driver.execute_script("arguments[0].click();", confirmation_button)
# Required to sleep the thread for a bit after using JS to click this button
time.sleep(5)
driver.refresh()
def get_web_driver(email,
password,
headless=False,
mfa_method=None,
mfa_input_callback=None,
wait_for_sync=True,
session_path=None,
imap_account=None,
imap_password=None,
imap_server=None,
imap_folder="INBOX"):
if headless and mfa_method is None:
warnings.warn(
"Using headless mode without specifying an MFA method"
"is unlikely to lead to a successful login. Defaulting --mfa-method=sms"
)
mfa_method = "sms"
zip_type = ""
executable_path = os.getcwd() + os.path.sep + 'chromedriver'
if _platform in ['win32', 'win64']:
executable_path += '.exe'
zip_type = CHROME_ZIP_TYPES.get(_platform)
if not os.path.exists(executable_path):
zip_file_url = CHROME_DRIVER_BASE_URL % (CHROME_DRIVER_VERSION,
zip_type)
request = requests.get(zip_file_url)
if request.status_code != 200:
raise RuntimeError(
'Error finding chromedriver at %r, status = %d' %
(zip_file_url, request.status_code))
zip_file = zipfile.ZipFile(io.BytesIO(request.content))
zip_file.extractall()
os.chmod(executable_path, 0o755)
chrome_options = ChromeOptions()
if headless:
chrome_options.add_argument('headless')
chrome_options.add_argument('no-sandbox')
chrome_options.add_argument('disable-dev-shm-usage')
chrome_options.add_argument('disable-gpu')
# chrome_options.add_argument("--window-size=1920x1080")
if session_path is not None:
chrome_options.add_argument("user-data-dir=%s" % session_path)
driver = Chrome(chrome_options=chrome_options,
executable_path="%s" % executable_path)
driver.get("https://www.mint.com")
driver.implicitly_wait(20) # seconds
try:
element = driver.find_element_by_link_text("Log In")
except NoSuchElementException:
# when user has cookies, a slightly different front page appears
driver.implicitly_wait(0) # seconds
element = driver.find_element_by_link_text("LOG IN")
driver.implicitly_wait(20) # seconds
element.click()
time.sleep(1)
driver.find_element_by_id("ius-userid").send_keys(email)
driver.find_element_by_id("ius-password").send_keys(password)
driver.find_element_by_id("ius-sign-in-submit-btn").submit()
# Wait until logged in, just in case we need to deal with MFA.
while not driver.current_url.startswith(
'https://mint.intuit.com/overview.event'):
time.sleep(1)
# bypass "Let's add your current mobile number" interstitial page
try:
skip_for_now = driver.find_element_by_id(
'ius-verified-user-update-btn-skip')
skip_for_now.click()
except:
pass
driver.implicitly_wait(1) # seconds
try:
driver.find_element_by_id('ius-mfa-options-form')
try:
mfa_method_option = driver.find_element_by_id(
'ius-mfa-option-{}'.format(mfa_method))
mfa_method_option.click()
mfa_method_submit = driver.find_element_by_id(
"ius-mfa-options-submit-btn")
mfa_method_submit.click()
if mfa_method == 'email' and imap_account:
mfa_code = get_email_code(imap_account,
imap_password,
imap_server,
imap_folder=imap_folder)
else:
mfa_code = (mfa_input_callback or
input)("Please enter your 6-digit MFA code: ")
mfa_code_input = driver.find_element_by_id(
"ius-mfa-confirm-code")
mfa_code_input.send_keys(mfa_code)
mfa_code_submit = driver.find_element_by_id(
"ius-mfa-otp-submit-btn")
mfa_code_submit.click()
except Exception: # if anything goes wrong for any reason, give up on MFA
mfa_method = None
warnings.warn("Giving up on handling MFA. Please complete "
"the MFA process manually in the browser.")
except NoSuchElementException:
pass
finally:
driver.implicitly_wait(20) # seconds
# Wait until the overview page has actually loaded, and if wait_for_sync==True, sync has completed.
if wait_for_sync:
status_message = driver.find_element_by_css_selector(
".SummaryView .message")
try:
WebDriverWait(driver, 5 *
60).until(lambda x: "Account refresh complete" in
status_message.get_attribute('innerHTML'))
except TimeoutException:
warnings.warn(
"Mint sync apparently incomplete after 5 minutes. Data "
"retrieved may not be current.")
else:
driver.find_element_by_id("transaction")
return driver
def get_web_driver(email,
password,
headless=False,
mfa_method=None,
mfa_input_callback=None,
wait_for_sync=True):
if headless and mfa_method is None:
warnings.warn(
"Using headless mode without specifying an MFA method"
"is unlikely to lead to a successful login. Defaulting --mfa-method=sms"
)
mfa_method = "sms"
zip_type = ""
executable_path = os.getcwd() + os.path.sep + 'chromedriver'
if _platform in ['win32', 'win64']:
executable_path += '.exe'
zip_type = CHROME_ZIP_TYPES.get(_platform)
if not os.path.exists(executable_path):
zip_file_url = CHROME_DRIVER_BASE_URL % (CHROME_DRIVER_VERSION,
zip_type)
request = requests.get(zip_file_url)
if request.status_code != 200:
raise RuntimeError(
'Error finding chromedriver at %r, status = %d' %
(zip_file_url, request.status_code))
zip_file = zipfile.ZipFile(io.BytesIO(request.content))
zip_file.extractall()
os.chmod(executable_path, 0o755)
chrome_options = ChromeOptions()
if headless:
chrome_options.add_argument('headless')
chrome_options.add_argument('no-sandbox')
chrome_options.add_argument('disable-dev-shm-usage')
chrome_options.add_argument('disable-gpu')
# chrome_options.add_argument("--window-size=1920x1080")
driver = Chrome(chrome_options=chrome_options,
executable_path="%s" % executable_path)
driver.get(
"https://accounts.intuit.com/index.html?redirect_url=https%3A%2F%2Fmint.intuit.com%2Foverview.event"
)
driver.implicitly_wait(20) # seconds
driver.find_element_by_id("ius-userid").send_keys(email)
driver.find_element_by_id("ius-password").send_keys(password)
driver.find_element_by_id("ius-sign-in-submit-btn").submit()
# Wait until logged in, just in case we need to deal with MFA.
while not driver.current_url.startswith(
'https://mint.intuit.com/overview.event'):
time.sleep(1)
driver.implicitly_wait(1) # seconds
try:
try:
if mfa_method == "app":
# mfa_input_callback should be a function that returns the 2fa barcode.
two_factor_code = mfa_input_callback()
driver.find_element_by_id("ius-mfa-soft-token").send_keys(
two_factor_code)
driver.find_element_by_id(
"ius-mfa-soft-token-submit-btn").submit()
else:
driver.find_element_by_id('ius-mfa-options-form')
mfa_method_option = driver.find_element_by_id(
'ius-mfa-option-{}'.format(mfa_method))
mfa_method_option.click()
mfa_method_submit = driver.find_element_by_id(
"ius-mfa-options-submit-btn")
mfa_method_submit.click()
mfa_code = (mfa_input_callback or
input)("Please enter your 6-digit MFA code: ")
mfa_code_input = driver.find_element_by_id(
"ius-mfa-confirm-code")
mfa_code_input.send_keys(mfa_code)
mfa_code_submit = driver.find_element_by_id(
"ius-mfa-otp-submit-btn")
mfa_code_submit.click()
except NoSuchElementException:
pass
except Exception as e:
mfa_method = None
warnings.warn(
"Giving up on handling MFA. Please complete "
"the MFA process manually in the browser. Exception: " +
repr(e))
except NoSuchElementException:
pass
finally:
driver.implicitly_wait(20) # seconds
# Wait until the overview page has actually loaded, and if wait_for_sync==True, sync has completed.
if wait_for_sync:
try:
driver.implicitly_wait(5)
status_message = driver.find_element_by_css_selector(
".SummaryView .message")
WebDriverWait(driver, 5 *
60).until(lambda x: "Account refresh complete" in
status_message.get_attribute('innerHTML'))
except TimeoutException:
warnings.warn(
"Mint sync apparently incomplete after 5 minutes. Data "
"retrieved may not be current.")
else:
driver.find_element_by_id("transaction")
return driver
class DigitalCommonsConnection:
def __init__(self, user, password):
self.options = Options()
self.driver = Chrome(
executable_path=os.path.abspath("/usr/bin/chromedriver"),
options=self.options)
self.login(user, password)
self.dissertations = self.get_list_of_dissertations()
self.lookup_values = self.__review_dissertations()
self.__lookup_decisions()
def login(self, username, passwd):
self.driver.get(
'https://trace.tennessee.edu/cgi/myaccount.cgi?context=')
self.driver.find_element_by_id('auth_email').send_keys(username)
self.driver.find_element_by_id('auth_password').send_keys(passwd)
self.driver.find_element_by_xpath(
'/html/body/div[2]/div/div[3]/div[1]/div[1]/div/div[2]/div[1]/div/form/div/p/button'
).click()
def get_list_of_dissertations(self):
self.driver.get(
'https://trace.tennessee.edu/utk_graddiss/index.11.html#year_2015')
disserations = self.driver.find_elements_by_css_selector(
'.article-listing > a')
return [
disserations[link].get_attribute('href')
for link in range(0, len(disserations))
]
def __review_dissertations(self):
lookups = []
for dissertation in self.dissertations:
self.driver.get(dissertation)
link = self.driver.find_element_by_css_selector('#title > p > a')
lookups.append(
link.get_attribute('href').split('=')[1].split('&')[0])
return lookups
def __lookup_decisions(self):
for dissertation in self.lookup_values:
self.driver.get(
f'https://trace.tennessee.edu/cgi/editor.cgi?article={dissertation}'
f'&window=viewdecisions&context=utk_graddiss')
decisions = self.driver.find_elements_by_css_selector(
'.MenuMain > tbody > tr > td > table > tbody > tr > td > a')
all_decisions = [
decisions[link].get_attribute('href')
for link in range(0, len(decisions)) if link !=
'https://trace.tennessee.edu/cgi/help.cgi?context=utk_graddiss&help=help-submissions.html#'
]
for decision in all_decisions:
self.driver.get(decision)
try:
final_decision_metadata = self.driver.find_element_by_css_selector(
'.MenuMain > tbody > tr > td > span')
decision_metadata = final_decision_metadata.text
print(decision_metadata.split('\n'))
final_decision = self.driver.find_element_by_css_selector(
'.MenuMain > tbody > tr > td > pre')
except NoSuchElementException:
pass
return
