def test_add_indicator_to_iList(self):
login(self.driver)
create_iList(self.driver)
self.driver.refresh()
'''since we know the filter works from TestDataCatalogSearch,
we don't need to search filters again here. This will avoid
having to use WebDriverWait() which can sometimes cause
conflict with ajax requests.
'''
self.driver.get(S.CATALOG_FILTER_URL)
indicator_from_search_elem = self.driver.find_element_by_css_selector('.odd a')
indicator_from_search = indicator_from_search_elem.text
indicator_from_search_ckbx = self.driver.find_element_by_css_selector('.odd input')
indicator_from_search_ckbx.click()
add_btn = self.driver.find_element_by_id('indicator_list_actions_add_to_ilist')
add_btn.click()
self.driver.get(S.USER_PORTFOLIO_URL)
self.driver.refresh()
iList_to_edit = WebDriverWait(self.driver, 10).until(EC.visibility_of(self.driver.find_element_by_css_selector('.edit_ilist')))
iList_to_edit.click()
indicator_in_iList_elem = WebDriverWait(self.driver, 10).until(EC.visibility_of(self.driver.find_element_by_id('indicator_2362')))
indicator_in_iList = indicator_in_iList_elem.text
self.assertEqual(indicator_from_search, indicator_in_iList)
sign_out(self.driver)
def test_chrome_app_link():
# The web driver for Safari does not yet support the move_to_element method so this test will not function properly
if (driver.capabilities['browserName'] == "safari"):
raise SkipTest
driver.get(testcenter_url + 'en')
button = driver.find_element_by_xpath('//div[@id="app"]/section[1]/div[1]/div[1]/div[1]/div/div')
hoverButton = ActionChains(driver).move_to_element(button)
hoverButton.perform()
loginBox = driver.find_element_by_xpath('//div[@id="app"]/section[1]/div[1]/div[2]/p')
chromeLink = driver.find_element_by_css_selector('.start-chrome')
wait = WebDriverWait(driver, 30)
wait.until(EC.visibility_of(chromeLink))
if chromeLink.is_displayed():
chromeLink.click()
# Users are redirected to a user creation page if they are not currently logged in when using Chrome
# If they are using other browsers, they are redirected to a Chrome download page
if driver.capabilities['browserName'] == 'chrome':
wait = WebDriverWait(driver, 30)
wait.until(EC.visibility_of(loginBox))
loginText = driver.find_element_by_xpath('//div[@id="app"]/section[1]/div[1]/div[2]/p').text.strip()
assert (loginText == "You need a Duolingo account to save your test results.")
else:
try:
elem = driver.find_element_by_xpath("//*[contains(.,'Download Chrome')]")
assert True
except:
assert False
def login_user(self):
"""
Taken from account views tests
FIXME: Ideally this can be deleted and user authenitcation can happen in setUp
"""
# Login user
# 1. Go to home page and confirm successful load
home_url = reverse('index')
self.browser.get(self.live_server_url + home_url)
actual = self.browser.current_url
expected = self.live_server_url + u'/'
self.assertEqual(actual, expected)
# 2. Click 'Login' modal link in header
login_link = self.browser.find_element_by_css_selector(
'p[data-target="#Login-Modal"]')
login_link.click()
# 3. Confirm Login modal appears
login_modal = self.browser.find_element_by_css_selector('#Login-Modal')
wait = WebDriverWait(self.browser, 10)
element = wait.until(EC.visibility_of(login_modal))
actual = login_modal.is_displayed()
expected = True
self.assertEqual(actual, expected)
self.browser.save_screenshot('login_modal.png')
# 4. Enter email and password
email_input = self.browser.find_element_by_css_selector('#id_username')
password_input = self.browser.find_element_by_css_selector('#id_password')
wait = WebDriverWait(self.browser, 10)
element = wait.until(EC.visibility_of(password_input))
email_input.send_keys(self.user_email)
password_input.send_keys(self.user_password)
password_input.submit()
# 5. Refresh page to show featured facilities
self.browser.save_screenshot('logged_in.png')
def test_mobile_home_page_aesthetics(self):
self.browser.get(self.server_url)
# Basic intro text is displayed
self.assert_text_in_body(
'Search millions of opinions by case name, topic, or citation.'
)
# on mobile, the navbar should start collapsed into a hamburger-esque
# icon in the upper right
navbar_header = (self.browser
.find_element_by_css_selector('.navbar-header'))
navbtn = navbar_header.find_element_by_tag_name('button')
self.assertIn('collapsed', navbtn.get_attribute('class'))
self.assertAlmostEqual(
navbtn.location['x'] + navbtn.size['width'] + 10,
MOBILE_WINDOW[0],
delta=10
)
# clicking the button displays and then hides the menu
menu = self.browser.find_element_by_css_selector('.nav')
self.assertFalse(menu.is_displayed())
navbtn.click()
WebDriverWait(self.browser, 5).until(EC.visibility_of(menu))
self.assertTrue(menu.is_displayed())
# and the menu is width of the display
self.assertAlmostEqual(
menu.size['width'],
MOBILE_WINDOW[0],
delta=5
)
# and the menu hides when the button is clicked
navbtn.click()
WebDriverWait(self.browser, 5).until_not(EC.visibility_of(menu))
self.assertFalse(menu.is_displayed())
# search box should always be centered
searchbox = self.browser.find_element_by_id('id_q')
search_button = self.browser.find_element_by_id('search-button')
juri_select = self.browser.find_element_by_css_selector(
'div[data-content="Select Jurisdictions"]'
)
search_width = (searchbox.size['width'] +
search_button.size['width'] +
juri_select.size['width'])
self.assertAlmostEqual(
searchbox.location['x'] + search_width / 2,
MOBILE_WINDOW[0] / 2,
delta=10
)
# and the search box should be ~250px wide in mobile layout
self.assertAlmostEqual(
searchbox.size['width'],
250,
delta=5
)
def register(self):
"registers as test/[email protected]/test"
driver = self.driver
with Annotator(driver):
# Find the signin link and click it
signin = driver.find_element_by_link_text("Sign in")
ec = expected_conditions.visibility_of(signin)
WebDriverWait(driver, 30).until(ec)
signin.click()
# Find the authentication form sheet
auth = driver.find_element_by_class_name('sheet')
# Switch to the registration tab
auth.find_element_by_link_text("Create an account").click()
# Get the registration pane
form = auth.find_element_by_name('register')
ec = expected_conditions.visibility_of(form)
WebDriverWait(driver, 30).until(ec)
username = form.find_element_by_name('username')
username.send_keys("test")
email = form.find_element_by_name('email')
email.send_keys("*****@*****.**")
password = form.find_element_by_name('password')
password.send_keys("test")
form.submit()
picker = (By.CLASS_NAME, 'user-picker')
ec = expected_conditions.visibility_of_element_located(picker)
WebDriverWait(self.driver, 30).until(ec)
def test_contact_key_creator(self):
self.logger.info('Starting test_contact_key_creator')
#setup the doc on gdrive first
file_name = 'ContactKeyTest1'
ss_key = gdoc_util.upload_file_to_gdrive('contact_key_test1.tsv', file_name)
driver = self.driver
gdoc_util.login_gdrive(driver)
driver.get('%s%s' % (self.base_url, '?ss=' + ss_key))
gc = gspread.login(settings.DEFAULT_GDRIVE_EMAIL, settings.DEFAULT_GDRIVE_PW)
my_worksheet = gc.open_by_key(ss_key).sheet1
e2_val = my_worksheet.acell('E2')
self.logger.info('e2_val: %s' % e2_val)
#reset the cell
my_worksheet.update_acell('E2', '')
e2_val = my_worksheet.acell('E2')
self.logger.info('e2_val reset to: %s' %e2_val)
#now run the command
#switch to input form frame
driver.switch_to.frame(0)
WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.LINK_TEXT, "Hiplead"))
).click()
id_worksheet_name_INPUT = WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.ID, "id_worksheet_name"))
)
id_worksheet_name_INPUT.clear()
id_worksheet_name_INPUT.send_keys(file_name)
Select(driver.find_element_by_id("id_scrapers")).select_by_value('contactKeyCreator')
driver.find_element_by_id("id_email").send_keys(settings.DEFAULT_GDRIVE_EMAIL)
driver.find_element_by_id("id_password").send_keys(settings.DEFAULT_GDRIVE_PW)
#ok, now submit the form
id_worksheet_name_INPUT.submit()
#then wait for task to complete
#this success alert only becomes visible when task is actually finished.
success_div = driver.find_element_by_class_name('time_remaining')
try:
WebDriverWait(driver, 10).until(
EC.visibility_of(success_div)
)
except StaleElementReferenceException as e:
#TODO The javascript DOM manipulation that results in StaleElementReferenceException needs to be resolved.
success_div = driver.find_element_by_class_name('time_remaining')
WebDriverWait(driver, 10).until(
EC.visibility_of(success_div)
)
#now validate cell value, since we know task has completed.
e2_val = my_worksheet.acell('E2')
self.logger.info('e2_val after test: %s' %e2_val)
self.assertEquals('john_franklin_smith_somedomain.net', e2_val.value)
self.logger.info( 'Finished test_contact_key_creator')
def login(self):
"signs in as test/[email protected]/test"
driver = self.driver
with Annotator(driver):
# Find the signin link and click it
signin = driver.find_element_by_link_text("Sign in")
ec = expected_conditions.visibility_of(signin)
WebDriverWait(driver, 30).until(ec)
signin.click()
# Find the authentication form sheet
auth = driver.find_element_by_class_name('sheet')
# Find the login pane
form = auth.find_element_by_name('login')
ec = expected_conditions.visibility_of(form)
WebDriverWait(driver, 30).until(ec)
username = form.find_element_by_name('username')
username.send_keys("test")
password = form.find_element_by_name('password')
password.send_keys("test")
form.submit()
def test_delete_indicator_from_iList(self):
login(self.driver)
self.driver.get(S.USER_PORTFOLIO_URL)
iList_to_edit = WebDriverWait(self.driver, 10).until(EC.visibility_of(self.driver.find_element_by_css_selector('.edit_ilist')))
iList_to_edit.click()
indicator_ckbx = WebDriverWait(self.driver, 10).until(EC.visibility_of(self.driver.find_element_by_name("indicator_id")))
indicator_ckbx.click()
remove_selected_btn = self.driver.find_element_by_id("indicator_list_lightbox_remove_selected")
remove_selected_btn.click()
exit_lightbox_btn = self.driver.find_element_by_id("cboxClose")
exit_lightbox_btn.click()
self.driver.refresh()
iList_to_edit = WebDriverWait(self.driver, 10).until(EC.visibility_of(self.driver.find_element_by_css_selector('.edit_ilist')))
iList_to_edit.click()
empty_table_elem = WebDriverWait(self.driver, 10).until(EC.visibility_of(self.driver.find_element_by_class_name("dataTables_empty")))
empty_table_txt = empty_table_elem.text
self.assertEqual(empty_table_txt, 'No data available in table')
sign_out(self.driver)
def testExpectedConditionVisibilityOf(self, driver, pages):
pages.load("javascriptPage.html")
hidden = driver.find_element_by_id('clickToHide')
with pytest.raises(TimeoutException):
WebDriverWait(driver, 0.7).until(EC.visibility_of(hidden))
driver.find_element_by_id('clickToShow').click()
element = WebDriverWait(driver, 5).until(EC.visibility_of(hidden))
assert element.is_displayed() is True
def test_accordion_minimize_by_double_click(self):
"""Accordion item should be minimized by two clicks on title."""
accordion_title = self.accordion_title
accordion_content = self.accordion_content
accordion_title.click()
self.wait.until(EC.visibility_of(accordion_content))
accordion_title.click()
self.wait.until_not(EC.visibility_of(accordion_content))
self.assertFalse(accordion_content.is_displayed())
def testExpectedConditionVisibilityOf(self):
self._loadPage("javascriptPage")
hidden = self.driver.find_element_by_id('clickToHide')
try:
WebDriverWait(self.driver, 0.7).until(EC.visibility_of(hidden))
self.fail("Expected TimeoutException to have been thrown")
except TimeoutException as e:
pass
self.driver.find_element_by_id('clickToShow').click()
element = WebDriverWait(self.driver, 5).until(EC.visibility_of(hidden))
self.assertTrue(element.is_displayed())
def test_multi_stash(self):
self.login()
self.open(reverse('admin:test_app_testmodeladvanced_change', args=[self.advanced.id]))
inline = self.webdriver.find_css("#testinlinemodel_set-group")
wait = WebDriverWait(self.webdriver, 1)
wait.until(visibility_of(inline))
# self.assertTrue(inline.is_displayed())
f11 = self.webdriver.find_css("div.field-set1_1")
wait.until(visibility_of(f11))
# self.assertTrue(f11.is_displayed())
f31 = self.webdriver.find_css("div.field-set3_1")
wait.until(invisibility_of(f31))
def test_amvr(self):
driver = self.driver
waiting = self.waiting
driver.get("https://apps.tn.gov/amvr-app/login.html")
waiting.until(expected_conditions.title_is("Log In - Motor Vehicle Records Search"))
username_textbox = waiting.until(expected_conditions.visibility_of(driver.find_element_by_id("username")))
password_textbox = waiting.until(expected_conditions.visibility_of(driver.find_element_by_id("password")))
login_button = waiting.until(expected_conditions.visibility_of(driver.find_element_by_name("login")))
waiting.until(expected_conditions.title_is("Log In - Motor Vehicle Records Search"))
self.assertIn("amvr-app/login.html", driver.current_url, "Failed to login")
def search(self, search_term):
"""
Searches for an app using the available search field
:Args:
- search_term - string value of the search field
"""
search_toggle = self.selenium.find_element(*self._search_toggle_locator)
WebDriverWait(self.selenium, self.timeout).until(EC.visibility_of(search_toggle))
search_toggle.click()
search_field = self.selenium.find_element(*self._search_input_locator)
WebDriverWait(self.selenium, self.timeout).until(EC.visibility_of(search_field))
search_field.send_keys(search_term)
search_field.submit()
from pages.desktop.consumer_pages.search import Search
return Search(self.testsetup, search_term)
def getCounts(employer):
'''
takes the company name and returns the number of
applications submitted during the period between
the first and second elements of the dates obj
'''
global driver
if (len(driver.find_elements_by_css_selector('#visa_h1b1'))==0):
driver.back()
driver.implicitly_wait(3)
driver.find_element_by_id("visa_h1b1").click()
elem1 = driver.find_element_by_id('employer_business_name')
elem1.clear()
elem1.send_keys(employer)
elem2 = driver.find_element_by_id('start_date_from')
elem2.clear()
elem2.send_keys(dates[0])
elem3 = driver.find_element_by_id('start_date_to')
elem3.clear()
elem3.send_keys(dates[1])
driver.find_element_by_id('btnSearch_employment2').click()
elem = driver.find_element_by_id('numberOfCaseFound')
wait = WebDriverWait(driver, 5)
count = wait.until(EC.visibility_of(elem)).text
return int(count.replace(',',''))
def wait_for_visible_element(self,
web_element=None,
locator=None,
wait_period=None):
"""Pause until the element is displayed/visible
Selenium Driver will wait for the web element to be visible.
The web element can be provided as a Selenium Web Element or as
an Element Locator Tuple
:param web_element: Optional argument. Default value of None
:type web_element: Selenium Web Element Object
:param locator: Optional argument. Default value of None
:type locator: Tuple of two items: Locator Type and Locator String
:param wait_period: Optional argument. Seconds to wait for the Web
Element to be displayed. Default value of None
:type wait_period: int
"""
wait = self.ui_config.page_load_timeout if wait_period is None \
else wait_period
if web_element is not None:
WebDriverWait(
self.driver,
wait).until(
EC.visibility_of(web_element))
else:
WebDriverWait(
self.driver,
wait).until(
EC.visibility_of_element_located(locator))
def logout(self):
# If present, close the "new branding" overlay as it blocks logging out
elems = self.driver.find_elements_by_id("self.webklipper-publisher-widget-container-notification-close-div")
self.assertFalse(
len(elems) > 1,
"Found more than one element with the id: "
+ "webklipper-publisher-widget-container-notification-close-div",
)
if len(elems) == 1:
elems[0].click()
self.driver.get("http://" + self.DOMAIN + self.DASHBOARD)
try:
elem = self.wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, ".fn-hover-menu.span-7")))
except TimeoutException as te:
print(te.msg)
# check if logged in
self.assertTrue(
self.wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, "a[href='/accounts/login/']")))
)
return
self.assertTrue(elem is not None, "Failed to find hoverable element used to make logout link visible")
hover = ActionChains(self.driver).move_to_element(elem)
hover.perform()
elem = self.wait.until(
EC.visibility_of(self.driver.find_element_by_css_selector("a[href='/accounts/logout/']"))
)
elem.click()
self.elem = self.wait.until(EC.presence_of_element_located((By.XPATH, "//a[@href='/accounts/login/']")))
def test_notification_dismiss(self):
"""
Test that when we dismiss notification they are hidden but that the dropdown stays open
so we can dismiss several of them in a row.
"""
self.check_notification_count(1)
for i in range(0, 10):
self.create_notification('foo%s' % i)
time.sleep(2)
self.check_notification_count(11)
notif_xpath = '//a[@id="notifications_dropdown_button"]'
notification_icon = self.selenium.find_element_by_xpath(notif_xpath)
ActionChains(self.selenium).move_to_element(notification_icon).perform()
dismiss_all_xpath = '//div[@class="notification_dismiss"]'
dismiss_xpath = '(%s)[1]' % dismiss_all_xpath
WebDriverWait(self.selenium, 4).until(
EC.visibility_of(self.selenium.find_element_by_xpath(dismiss_xpath)))
for i in range(11, 0, -1):
time.sleep(1.5)
# Test pagination
self.assertEqual(min(5, i),\
len(self.selenium.find_elements_by_xpath(dismiss_all_xpath)))
self.check_notification_count(i)
self.dismiss_notification()
time.sleep(1)
with self.assertRaises(NoSuchElementException):
self.selenium.find_element_by_xpath(dismiss_xpath)
def _remove_test_user_from_db(self):
self.driver.get(S.ADMIN_LOGIN_PAGE)
admin_username = self.driver.find_element_by_id('id_username')
admin_username.send_keys(S.SUPERUSER_USERNAME)
admin_password = self.driver.find_element_by_id('id_password')
admin_password.send_keys(S.SUPERUSER_PASSWORD)
admin_password.send_keys(Keys.RETURN)
#this searches the test username directly into url, less complications w/selenium
self.driver.get(S.USERNAME_SEARCH_URL)
username_link = WebDriverWait(self.driver, 10).until(EC.visibility_of(self.driver.find_element_by_link_text('testtestspaceorg')))
#ensures that this test name is actually in the db
self.assertEqual(username_link.text,'testtestspaceorg')
username_link.click()
delete_btn = self.driver.find_element_by_class_name('deletelink')
delete_btn.click()
confirm_btn = self.driver.find_element_by_css_selector('input[type="submit"]')
confirm_btn.click()
logout_btn = self.driver.find_element_by_css_selector('a[href="/admin/logout/"]')
logout_btn.click()
def public(self, value):
if self.public == value:
return
# If public, the "Make private" element will be the only <a>.
# If private, the opposite is true.
self.driver.find_element_by_css_selector(
'#overview div.btn-group:nth-of-type(1) > a'
).click()
WebDriverWait(self.driver, 3).until(
EC.visibility_of_element_located(
(By.CSS_SELECTOR, 'div.modal.fade.in button.btn-primary')
)
)
confirm_button = self.driver.find_element_by_css_selector(
'div.modal.fade.in button.btn-primary'
)
WebDriverWait(self.driver, 1).until(
EC.visibility_of(confirm_button)
)
with WaitForPageReload(self.driver):
confirm_button.click()
def remove_test_user(self, name, server_url, should_raise_exception=True):
"""Manually remove a test user using the landing page (the only way).
Args:
name: A string for the name of the test user to remove.
server_url: The base url portion of the landing page.
should_raise_exception: True to raise an exception if the user is not
found.
"""
self.go_to_landing_page(server_url)
# Find the user and navigate to their details page.
user_item = self.findTestUser(name)
if user_item is None:
if should_raise_exception:
raise Exception
else:
return
else:
user_item.click()
# Click delete on that user.
details_modal = user_item.find_element(*LandingPage.DETAILS_MODAL)
WebDriverWait(self.driver, UfOPageLayout.DEFAULT_TIMEOUT).until(
EC.visibility_of(details_modal))
delete_button = details_modal.find_element(*LandingPage.USER_DELETE_BUTTON)
delete_button.click()
# Wait for post to finish, can take a while.
WebDriverWait(self.driver, UfOPageLayout.DEFAULT_TIMEOUT).until(
EC.invisibility_of_element_located(((
LandingPage.USER_DETAILS_SPINNER))))
def locate_text_by_text(context, text):
element = context.browser, context.browser.find_element_by_xpath('//*[contains(text(), "%s")]' % text)
WebDriverWait(context.browser, TIME_FOR_WAIT).until(
EC.visibility_of(element)
)
scroll_element_into_view(context.browser, element)
assert context.browser.find_element_by_xpath('//*[contains(text(), "%s")]' % text)
def post(self,meg):
post_button_xpath = '''//*[@id="global-new-tweet-button"]'''
input_xpath = '''//*[@id="tweet-box-global"]'''
send_xpath = '''//*[@id="global-tweet-dialog-dialog"]/div[2]/div[4]/form/div[2]/div[2]/button'''
first_li_xpath = '''//*[@id="stream-items-id"]/li[1]'''
post_button = self.driver.find_element_by_xpath(post_button_xpath)
post_button.click()
input_element = self.driver.find_element_by_xpath(input_xpath)
try:
WebDriverWait(self.driver, 10).until(EC.visibility_of(input_element))
except Exception:
logging.info(input_element.get_attribute("class"))
logging.error("error when wait input_element visibility.")
return
input_element.clear()
input_element.send_keys("")
input_element.send_keys(meg)
try:
send_element = WebDriverWait(self.driver, 10).until(EC.element_to_be_clickable((By.XPATH, send_xpath)))
except Exception, e:
print e
logging.info(input_element.get_attribute("class"))
logging.error("error when wait input_element visibility.")
return
def __init__(self, url, driver):
driver.get(url)
self.num_entries = Select(driver.find_element_by_id("numentries"))
form = driver.find_element_by_id("form")
assert not form.is_displayed()
self.set_num_entries(1)
WebDriverWait(driver, 10).until(EC.visibility_of(form))
assert form.is_displayed()
self.fathers_name = FullName(assertOne(form.find_elements_by_id("father-name")))
self.mothers_name = FullName(assertOne(form.find_elements_by_id("mother-name")))
self.address = Address(assertOne(form.find_elements_by_class_name("address")))
self.primary_email = Email(assertOne(form.find_elements_by_id("primary-email")))
self.secondary_email = Email(assertOne(form.find_elements_by_id("secondary-email")))
self.cell_phone_number = PhoneNumber(form.find_elements_by_class_name("phone-number")[0], "Cell")
self.home_phone_number = PhoneNumber(form.find_elements_by_class_name("phone-number")[1], "Home")
self.students = []
for i in range(1, 5):
self.students.append(Camper(assertOne(form.find_elements_by_id("camper-" + str(i)))))
self._submit = form.find_element_by_id("submit")
def wait_until_element_is_visible(driver, element, timeout):
"""Waits until a DOM element is visible within a given timeout.
Args:
driver: An instance of a Selenium webdriver browser class.
element: A Selenium webdriver element.
timeout: The maximum time to wait (in seconds).
Returns:
True if the element became visible within the timeout.
"""
try:
# Apparently checks for an element's visibility are broken in Safari 10.
# For addtional (though not much) information, see:
# http://stackoverflow.com/questions/40635371/selenium-3-0-1-with-safaridriver-failing-on-waitforelementvisible
# https://groups.google.com/forum/#!msg/selenium-users/xEGcK92rzVg/IboybWUPAAAJ
#
# To get around this for now, check the browser name and version (WebKit
# version , really), and if they seem to indicate Safari 10, then do a
# dumb wait of 'timeout' seconds to give the page time to render and the
# elements to become visible, etc.
if (driver.capabilities['browserName'] == 'safari' and
driver.capabilities['version'] == SAFARI10_VERSION):
time.sleep(timeout)
else:
ui.WebDriverWait(
driver,
timeout).until(expected_conditions.visibility_of(element))
except exceptions.TimeoutException:
return False
return True
def wait_for_element_visible(self, *args):
if len(args) == 1 and isinstance(args[0], WebElement):
Wait(self.selenium, self.timeout).until(
expected.visibility_of(args[0]))
else:
Wait(self.selenium, self.timeout).until(
expected.visibility_of_element_located(args))
def wait_until_element_is_visible(self, locator, timeout=None, error=None):
timeout = self._implicit_wait_in_secs if timeout is None else utils.timestr_to_secs(timeout)
if not error:
error = "Element '%s' was not visible in %s" % (locator, self._format_timeout(timeout))
element = self._element_find(locator, True, True)
if element is None:
raise AssertionError("Element '%s' not found." % locator)
WebDriverWait(None, timeout, self._poll_frequency).until(visibility_of(element), error)
def _wait_until_visible_then_click(self, element, scroll_up=True):
# See: http://stackoverflow.com/questions/23857145/selenium-python-element-not-clickable
element = WebDriverWait(self.browser, 5, poll_frequency=.2).until(
EC.visibility_of(element))
if scroll_up:
self.browser.execute_script("window.scrollTo(0, 0)")
element.click()
def test_preview_btn(self):
driver = self.driver
btn = driver.find_element_by_id('previewOpen')
self.assertTrue(btn.is_displayed())
btn.click()
wait = WebDriverWait(driver, 5)
wait.until(EC.visibility_of(driver.find_element_by_id('previewDialog')))
self.assertTrue(driver.find_element_by_id('previewDialog').is_displayed())
def car_type(self, response):
#click type
#self.driver.get is a must to use selenium after a call to url using scrapy
self.driver.get(response.url)
self.driver.implicitly_wait(5)
action = ActionChains(self.driver)
menu = self.driver.find_element_by_xpath('//div[contains(@data-reactid, "582")]')
WebDriverWait(self.driver,20).until(EC.visibility_of(menu))
action.click(menu).perform()
#click sedan type
menu1 = menu.find_element_by_xpath('//span[contains(@data-reactid, "698")]')
WebDriverWait(self.driver,20).until(EC.visibility_of(menu1))
menu1.click()
#click filter for distance search
action = ActionChains(self.driver)
filter = self.driver.find_element_by_xpath('//span[@data-reactid = "33"]')
WebDriverWait(self.driver,20).until(EC.visibility_of(filter))
action.click(filter).perform()
time.sleep(10)
#key up 8 times to get 25 miles
self.driver.find_element_by_class_name("Select-input").send_keys(Keys.ARROW_UP + Keys.ARROW_UP \
+ Keys.ARROW_UP + Keys.ARROW_UP + Keys.ARROW_UP + Keys.ARROW_UP + Keys.ARROW_UP + \
Keys.ARROW_UP + Keys.ARROW_UP + Keys.TAB)
time.sleep(10)
#call function to load all pages and load sedans
page = self.driver.find_element_by_xpath('//a[@class="pagination--next"]')
while page != []:
listing = self.driver.find_elements_by_xpath('//h3[@class="vehicle-browse--result-title tablet-hidden"]/a')
car_url=[]
for lists in listing:
car_url.append(lists.get_attribute('href'))
for each_car in range(0, len(car_url), 1):
yield scrapy.Request(car_url[each_car], callback=self.parse_listing_results_page)
self.count_car = self.count_car + len(car_url)
actions = ActionChains(self.driver)
page = self.driver.find_element_by_xpath('//a[@class="pagination--next"]')
if page != []:
actions.move_to_element(page).click().perform()
time.sleep(5)
def save_changes(self):
self.selenium.find_element(*self._save_button_locator).click()
notification = self.selenium.find_element(*self._notification_locator)
WebDriverWait(self.selenium,
self.timeout).until(EC.visibility_of(notification))
def wait_for_login_page_to_load(self):
wait = WebDriverWait(self.driver, 30)
wait.until(
expected_conditions.visibility_of(
self.driver.find_element_by_xpath(
"//*[@class='loggedout_menubar_container']")))
def get_two_fa_code(self) -> Union[str, None]:
code_two_fa = None
driver_2fa = new_chrome_driver(self.driver_path, name='google_msg')
if driver_2fa is None:
return None
driver_2fa.get('https://messages.google.com/web')
sms_auth_present = EC.presence_of_element_located((By.CLASS_NAME, _GOOG_QR_CODE_CLASS))
sms_code_present = EC.text_to_be_present_in_element((By.CSS_SELECTOR, _GOOG_MESSAGES_LIST_CLASS),
_GOOG_2FA_HEADING)
WebDriverWait(driver_2fa, 240).until(any_of(sms_auth_present, sms_code_present))
sms_auth_el = driver_2fa.find_elements_by_class_name(_GOOG_QR_CODE_CLASS)
if sms_auth_el:
driver_2fa.find_element_by_class_name(_GOOG_AUTH_REMEMBER_CLASS).click()
data = urllib.parse.quote(sms_auth_el[0].get_attribute('data-' + _GOOG_QR_CODE_CLASS))
_LOGGER.info(
'Web messages is not authenticated. Open this URL to pair web messages with your android phone:')
_LOGGER.info(
f'http://api.qrserver.com/v1/create-qr-code/?color=000000&bgcolor=FFFFFF&qzone=1&margin=0&size=400x400&ecc=L&data={data}')
WebDriverWait(driver_2fa, 120).until(sms_code_present)
sms_list_el = driver_2fa.find_elements_by_css_selector(_GOOG_MESSAGES_LIST_CLASS)
if not sms_list_el:
_LOGGER.error('Timeout or authentication error while loading sms messages.')
save_screenshot(driver_2fa, postfix='__google_2fa')
else:
_LOGGER.info(sms_list_el[0].text)
code_two_fa = re.search(r'(\d+)', sms_list_el[0].text).group(1)
_LOGGER.debug('Waiting for SMS message to be visible')
WebDriverWait(driver_2fa, 30).until(EC.visibility_of(sms_list_el[0]))
clicked_ok = False
for i in range(_GOOG_MESSAGE_CLICK_RETRIES):
try:
sms_list_el[0].click() # mark message as read
clicked_ok = True
_LOGGER.debug('SMS message marked as read')
break
except ElementClickInterceptedException as e:
if isinstance(e, ElementClickInterceptedException) \
and 'Other element would receive the click' in str(e):
_LOGGER.warning(f'Failed marking SMS message as read due to obstructing elements')
else:
_LOGGER.exception(f'Exception while marking SMS message as read: {e}')
save_screenshot(driver_2fa, postfix='__google_2fa')
_LOGGER.debug(f'Retrying clicking SMS message {_GOOG_MESSAGE_CLICK_RETRIES - i - 1} more times.')
time.sleep(2)
if not clicked_ok:
_LOGGER.warning('Failed all attempts to mark SMS message as read')
time.sleep(2) # wait for click to mark message as read
release_chrome_driver(driver_2fa)
return code_two_fa
def crawl_airbnb():
try:
config = read_config("config.toml")
except FileNotFoundError:
logger.exception("", exc_info=True)
raise
except Exception:
logger.exception("", exc_info=True)
raise
options = Options()
# options.add_argument("-headless")
browser = Firefox(executable_path=GECKODRIVER, options=options)
browser.implicitly_wait(30)
wait = WebDriverWait(browser, timeout=15)
actions = ActionChains(browser)
browser.get("https://www.airbnb.com/")
logger.debug("navigating to https://www.airbnb.com/")
time.sleep(2)
# click on search bar to choose location
try:
location_input_element = wait.until(
expected.visibility_of(
browser.find_element_by_id(
"Koan-magic-carpet-koan-search-bar__input")),
"",
)
except NoSuchElementException:
logger.exception("Could not find location input element.",
exc_info=True)
raise
else:
logger.debug("Found location input search bar.")
location_input_element.send_keys(config["location"])
logger.debug(f'Sent the location: {config["location"]} with send keys')
time.sleep(2)
try:
first_location_option = browser.find_element_by_id(
"Koan-magic-carpet-koan-search-bar__option-0")
except NoSuchElementException:
logger.exception("Could not find first location option.",
exc_info=True)
raise
else:
# first_location_option.click() and actions.click() don't click for some reason
# so execute_script is necessary
browser.execute_script("arguments[0].click();", first_location_option)
logger.debug("Found first option for location search bar and clicked")
start_date = date.fromisoformat(config["start_date"])
end_date = date.fromisoformat(config["end_date"])
try:
assert start_date < end_date
except AssertionError:
logger.exception(
f'Start date doesn\'t precede the end date in the configuration\nstart_date: {config["start_date"]}, end_date: {config["end_date"]}',
exc_info=True,
)
raise
try:
assert date.today() < start_date
except AssertionError:
logger.exception(
f'start date shouldn\'t precede today\'s date\nstart_date: {config["start_date"]}, today: {date.today()}',
exc_info=True,
)
raise
# find month and year on airbnb calendar
# Use this to figure out which month of the calendar is currently visible
current_month = date.today()
# current month object allows me to increment the date by a month
# each time browser searches for the current month on the calendar
current_month = date(current_month.year, current_month.month, 1)
month_and_year_xpath = f'//strong[contains(text(), "{current_month:%B %Y}")]'
try:
month_and_year = browser.find_element_by_xpath(month_and_year_xpath)
except NoSuchElementException:
logger.exception(
"Couldn't find month and year strong element on airbnb calendar",
exc_info=True,
)
raise
else:
logger.debug("Found strong element that contains month and year")
time.sleep(3)
next_month_arrow_xpath = '//div[@class="_1h5uiygl" and @aria-label="Move forward to switch to the next month."]'
try:
next_month_arrow = browser.find_element_by_xpath(
next_month_arrow_xpath)
except NoSuchElementException:
logger.exception("Couldn't find next month arrow on calendar",
exc_info=True)
raise
else:
logger.debug("Found next month arrow on calendar")
while month_and_year.text != f"{start_date:%B %Y}":
# check to see if somehow program moved past the correct month and year
# this happened occasionally in testing
# month_and_year_text = month_and_year.text.split()
# if int(month_and_year_text[1]) > start_date.year:
# raise ValueError(f'Visible calendar year: {month_and_year_text[1]} is greater than start_date year: {start_date.year}')
# increment year if december isn't the start date month
if current_month.month == 12:
current_month = date(current_month.year + 1, 1, current_month.day)
else:
current_month = date(current_month.year, current_month.month + 1,
current_month.day)
next_month_arrow.click()
time.sleep(2)
current_month_and_year_on_calendar = month_and_year.text
month_and_year_xpath = f'//strong[contains(text(), "{current_month:%B %Y}")]'
try:
month_and_year = browser.find_element_by_xpath(
month_and_year_xpath)
except NoSuchElementException:
logger.exception(
f"Couldn't find month and year strong element on airbnb calendar:\nmonth and year on calendar:{current_month_and_year_on_calendar}, start date month and year: {start_date:%B %Y}",
exc_info=True,
)
raise
else:
logger.debug("Clicked next month arrow.")
logger.debug("Found start date year and month on calendar")
# find and click start date
start_date_element_xpath = f'//td[@aria-label="Choose {start_date:%A, %B {start_date.day}, %Y} as your start date. It\'s available."]'
try:
start_date_element = browser.find_element_by_xpath(
start_date_element_xpath)
except NoSuchElementException:
logger.exception(
"Couldn't find start date td element that specifies the day",
exc_info=True)
raise
else:
start_date_element.click()
logger.debug("Found and clicked correct date for start date")
time.sleep(3)
month_and_year = browser.find_element_by_xpath(month_and_year_xpath)
next_month_arrow_xpath = '//div[@class="_1h5uiygl" and @aria-label="Move forward to switch to the next month."]'
while month_and_year.text != f"{end_date:%B %Y}":
if current_month.month == 12:
current_month = date(current_month.year + 1, 1, current_month.day)
else:
current_month = date(current_month.year, current_month.month + 1,
current_month.day)
next_month_arrow.click()
time.sleep(2)
current_month_and_year_on_calendar = month_and_year.text
month_and_year_xpath = f'//strong[contains(text(), "{current_month:%B %Y}")]'
try:
month_and_year = browser.find_element_by_xpath(
month_and_year_xpath)
except NoSuchElementException:
logger.exception(
f"Couldn't find month and year strong element on airbnb calendar.\nmonth and year on calendar: {current_month_and_year_on_calendar}, end date month and year: {end_date:%B %Y}",
exc_info=True,
)
raise
else:
logger.debug("Clicked next month arrow.")
logger.debug("Found month and year for end date on calendar")
# find and click end date
end_date_element_xpath = f'//td[@aria-label="Choose {end_date:%A, %B {end_date.day}, %Y} as your end date. It\'s available."]'
try:
end_date_element = browser.find_element_by_xpath(
end_date_element_xpath)
except NoSuchElementException:
logger.exception("Couldn't find specific date number and month: ",
exc_info=True)
raise
else:
end_date_element.click()
logger.debug("Found the end date on calendar and clicked")
time.sleep(2)
# start search
home_search_button_xpath = (
'//button[@class="_1vs0x720" and @type="submit" and @aria-busy="false"]'
)
try:
home_search_button = browser.find_element_by_xpath(
home_search_button_xpath)
except NoSuchElementException:
logger.exception("Couldn't find search button that submits the form",
exc_info=True)
else:
home_search_button.click()
logger.debug(
"Found search/submit button for form on main page and clicked.")
entire_homes_xpath = '//img[@alt="Entire homes"]'
try:
entire_homes = wait.until(
expected.visibility_of(
browser.find_element_by_xpath(entire_homes_xpath)))
except NoSuchElementException:
# If entire home button is not found then stays button is clicked
logger.debug(
"Couldn't find entire homes button, trying to click stays button.")
stays_button_xpath = '//a[@class="_10l4eyf" and @aria-busy="false" and @data-veloute="explore-nav-card:/homes"]'
stays_button = wait.until(
expected.visibility_of(
browser.find_element_by_xpath(stays_button_xpath)))
stays_button.click()
logger.debug("Found and clicked on stays button.")
time.sleep(3)
# finds and clicks type of place filter
type_of_place_xpath = '//button[@class="_1i67wnzj" and @type="button" and @aria-controls="menuItemComponent-room_type"]'
type_of_place = wait.until(
expected.visibility_of(
browser.find_element_by_xpath(type_of_place_xpath)))
type_of_place.click()
logger.debug("Found and clicked on type place filter/button.")
time.sleep(2)
# finds and click entire home checkbox
entire_home_checkbox_id = (
"DynamicFilterCheckboxItem-Type_of_place-room_types-Entire_home/apt"
)
entire_home_checkbox = browser.find_element_by_id(
entire_home_checkbox_id)
entire_home_checkbox.click()
logger.debug("Found and clicked on entire home checkbox.")
# saves the new filter for type of place
save_type_of_place_button_xpath = (
'//button[@class="_b0ybw8s" and @type="button" and @aria-busy="false"]'
)
save_type_of_place_button = wait.until(
expected.visibility_of(
browser.find_element_by_xpath(
save_type_of_place_button_xpath)))
save_type_of_place_button.click()
logger.debug(
"Found and clicked save button after checking entire place checkbox"
)
else:
entire_homes.click()
logger.debug("Found and clicked entire homes button.")
time.sleep(3)
# click on price filter button
price_button = wait.until(
expected.visibility_of(
browser.find_element_by_xpath(
r'//button[@aria-controls="menuItemComponent-price_range"]')))
time.sleep(2)
price_button.click()
logger.debug("Found and clicked price filter button.")
time.sleep(2)
# change minimum price
price_filter_min = browser.find_element_by_id("price_filter_min")
for i in range(len(price_filter_min.get_attribute("value"))):
price_filter_min.send_keys(Keys.BACK_SPACE)
price_filter_min.send_keys(str(config["min_price"]))
logger.debug(f'Changed minimum price to: {config["min_price"]}')
time.sleep(2)
# change max price
price_filter_max = browser.find_element_by_id("price_filter_max")
for i in range(len(price_filter_max.get_attribute("value"))):
price_filter_max.send_keys(Keys.BACK_SPACE)
time.sleep(2)
price_filter_max.send_keys(str(config["max_price"]))
logger.debug(f'Changed maximum price to: {config["max_price"]}')
apply_price = wait.until(
expected.visibility_of(
browser.find_element_by_xpath(
'//button[@class="_b0ybw8s" and @type="button" and @aria-busy="false"]'
)))
apply_price.click()
logger.debug("Clicked apply price for new min and max prices.")
more_filters_xpath = '//button[@type="button" and @aria-controls="menuItemComponent-dynamicMoreFilters"]'
time.sleep(2)
more_filters_button = browser.find_element_by_xpath(more_filters_xpath)
more_filters_button.click()
logger.debug("Found and clicked on more filters button.")
superhost_filter_id = "DynamicFilterSwitchItem-other_options-superhost-true--label"
time.sleep(2)
superhost_filter = browser.find_element_by_id(superhost_filter_id)
superhost_filter.click()
logger.debug("Found and clicked superhost filter.")
show_stays_xpath = '//button[@class="_72kmbi0" and @type="button" and @aria-busy="false"]'
time.sleep(2)
show_stays_button = browser.find_element_by_xpath(show_stays_xpath)
show_stays_button.click()
logger.debug("Found and clicked on show stays button.")
# wait is necessary so that it gives the browser time to load the results before
# getting the page_source
time.sleep(20)
logger.debug("Waited 20 seconds for browser to load results.")
page_source = browser.page_source
logger.debug("Retrieved page source.")
listings = []
# loops until it goes through 5 pages of results or find 15 listings
# that match the listing criteria
for i in range(5):
new_listings = scrape(browser.page_source)
logger.debug(f"Retrieved page {i} source.")
for i, _ in enumerate(new_listings):
listings.append(new_listings[i])
if len(listings) == 15:
break
if len(listings) == 15:
break
next_page_selector = "li._r4n1gzb > a._1ip5u88"
try:
next_page_arrow = browser.find_element_by_css_selector(
next_page_selector)
except NoSuchElementException:
logger.debug("No more pages of listing results")
break
else:
next_page_arrow.click()
logger.debug("Found and clicked next page arrow")
time.sleep(10)
browser.quit()
logger.debug("Exited browser.")
return listings
def hover_over_portfolio(self):
ActionChains(self.driver).move_to_element(self.portfolio).perform()
self.portfolio_sub_menu = self.portfolio.find_element_by_xpath("./ul")
self.wait.until(expected_conditions.visibility_of(self.portfolio_sub_menu))
def click_mobile_section_title(self):
self.mobile_section.title.click()
self.wait.until(expected_conditions.visibility_of(self.mobile_section.section_body))
