def local_element(self, xpath):
return WebDriverWait(self.driver, 5, 0.3).until(
EC.presence_of_element_located((By.XPATH, xpath)))
def load_private_sheets(self):
self.driver.get(self.base_url + "/sheets/private")
WebDriverWait(self.driver, TEMPER).until(
element_to_be_clickable((By.CSS_SELECTOR, ".sheet")))
return self
# here we declare a list where we will store full URL, https://www.footlocker.com+href
product_urls_full = []
for product_url in product_urls:
try:
product_urls_full.append("https://www.footlocker.com" +
product_url['href'])
except KeyError:
pass
# pass the product URL 1 by 1
for product_url_full in product_urls_full:
driver = webdriver.Chrome("/Users/ak054561/chromedriver")
driver.get(product_url_full)
wait = WebDriverWait(driver, 30)
html = driver.page_source
soup = bs(html, "html.parser")
print("URL Requested" + product_url_full)
span_tag = soup.find('span', attrs={"class": "ProductName-primary"})
print("Product Name: " + span_tag.text)
span_tag = soup.find('span', attrs={"class": "ProductPrice-original"})
print("Product Original: " + span_tag.text)
span_tag = soup.find('span', attrs={"class": "ProductPrice-final"})
print("Product Final: " + span_tag.text)
time.sleep(10)
def load_sheets(self):
self.driver.get(self.base_url + "/sheets")
WebDriverWait(self.driver, TEMPER).until(
element_to_be_clickable((By.CSS_SELECTOR, ".readerSheetsNav")))
self.set_modal_cookie()
return self
def load_account(self):
self.driver.get(self.base_url + "/account")
WebDriverWait(self.driver, TEMPER).until(
element_to_be_clickable(
(By.CSS_SELECTOR, ".accountPanel .blockLink")))
return self
def get_web_driver(email, password, headless=False, mfa_method=None,
mfa_input_callback=None, wait_for_sync=True,
session_path=None, imap_account=None, imap_password=None,
imap_server=None, imap_folder="INBOX"):
if headless and mfa_method is None:
warnings.warn("Using headless mode without specifying an MFA method"
"is unlikely to lead to a successful login. Defaulting --mfa-method=sms")
mfa_method = "sms"
zip_type = ""
executable_path = os.getcwd() + os.path.sep + 'chromedriver'
if _platform in ['win32', 'win64']:
executable_path += '.exe'
zip_type = CHROME_ZIP_TYPES.get(_platform)
if not os.path.exists(executable_path):
zip_file_url = CHROME_DRIVER_BASE_URL % (CHROME_DRIVER_VERSION, zip_type)
request = requests.get(zip_file_url)
if request.status_code != 200:
raise RuntimeError('Error finding chromedriver at %r, status = %d' %
(zip_file_url, request.status_code))
zip_file = zipfile.ZipFile(io.BytesIO(request.content))
zip_file.extractall()
os.chmod(executable_path, 0o755)
chrome_options = ChromeOptions()
if headless:
chrome_options.add_argument('headless')
chrome_options.add_argument('no-sandbox')
chrome_options.add_argument('disable-dev-shm-usage')
chrome_options.add_argument('disable-gpu')
# chrome_options.add_argument("--window-size=1920x1080")
if session_path is not None:
chrome_options.add_argument("user-data-dir=%s" % session_path)
driver = Chrome(chrome_options=chrome_options, executable_path="%s" % executable_path)
driver.get("https://www.mint.com")
driver.implicitly_wait(20) # seconds
try:
element = driver.find_element_by_link_text("Log In")
except NoSuchElementException:
# when user has cookies, a slightly different front page appears
driver.implicitly_wait(0) # seconds
element = driver.find_element_by_link_text("LOG IN")
driver.implicitly_wait(20) # seconds
element.click()
time.sleep(1)
driver.find_element_by_id("ius-userid").send_keys(email)
driver.find_element_by_id("ius-password").send_keys(password)
driver.find_element_by_id("ius-sign-in-submit-btn").submit()
# Wait until logged in, just in case we need to deal with MFA.
while not driver.current_url.startswith(
'https://mint.intuit.com/overview.event'):
time.sleep(1)
driver.implicitly_wait(1) # seconds
try:
driver.find_element_by_id('ius-mfa-options-form')
try:
mfa_method_option = driver.find_element_by_id('ius-mfa-option-{}'.format(mfa_method))
mfa_method_option.click()
mfa_method_submit = driver.find_element_by_id("ius-mfa-options-submit-btn")
mfa_method_submit.click()
if mfa_method == 'email' and imap_account:
mfa_code = get_email_code(imap_account, imap_password, imap_server, imap_folder=imap_folder)
else:
mfa_code = (mfa_input_callback or input)("Please enter your 6-digit MFA code: ")
mfa_code_input = driver.find_element_by_id("ius-mfa-confirm-code")
mfa_code_input.send_keys(mfa_code)
mfa_code_submit = driver.find_element_by_id("ius-mfa-otp-submit-btn")
mfa_code_submit.click()
except Exception: # if anything goes wrong for any reason, give up on MFA
mfa_method = None
warnings.warn("Giving up on handling MFA. Please complete "
"the MFA process manually in the browser.")
except NoSuchElementException:
pass
finally:
driver.implicitly_wait(20) # seconds
# Wait until the overview page has actually loaded, and if wait_for_sync==True, sync has completed.
if wait_for_sync:
status_message = driver.find_element_by_css_selector(".SummaryView .message")
try:
WebDriverWait(driver, 5 * 60).until(
lambda x: "Account refresh complete" in status_message.get_attribute('innerHTML')
)
except TimeoutException:
warnings.warn("Mint sync apparently incomplete after 5 minutes. Data "
"retrieved may not be current.")
else:
driver.find_element_by_id("transaction")
return driver
def _perform_segment_click(self, ref):
selector = '.segment[data-ref="{}"]'.format(ref.normal())
WebDriverWait(self.driver, TEMPER).until(
presence_of_element_located((By.CSS_SELECTOR, selector)))
segment = self.driver.find_element_by_css_selector(selector)
segment.click()
def getElementById(self, Time, cssId):
element = WebDriverWait(
self.driver, Time).until(lambda x: x.find_element_by_id(cssId))
return element
def getElementByClass(self, Time, cssClass):
element = WebDriverWait(
self.driver,
Time).until(lambda x: x.find_element_by_class_name(cssClass))
return element
def pause(selenium_browser, seconds_required=1.0):
start_time = time.time()
WebDriverWait(selenium_browser, seconds_required).until(
_when_done(start_time, seconds_required))
def run_inputs(instance_id, base_name, base_page):
print('Got bp: ' + str(base_page))
instance_status = {}
lp_status = 0
entered_chat = False
instance_status[str(base_name + str(instance_id))] = {}
instance_status[str(base_name + str(instance_id))]['timestamps'] = {}
instance_status[str(
base_name + str(instance_id))]['timestamps']['0_instance_born'] = str(
datetime.utcnow())
try:
if sys.platform != 'darwin' and use_xvfb:
print('Starting Xvfb')
vdisplay = Xvfb(width=1280, height=740)
vdisplay.start()
chrome_options = webdriver.ChromeOptions()
if sys.platform != 'darwin':
chrome_options.add_argument('headless')
chrome_options.add_argument('no-sandbox')
# chrome_options.add_argument('window-size=1200x700')
print('Will create driver')
driver = webdriver.Chrome(chrome_options=chrome_options)
print('driver:' + str(driver))
# os.environ['MOZ_HEADLESS'] = '1'
# driver = webdriver.Firefox()
except BaseException as e:
instance_status[str(base_name +
str(instance_id))]['status'] = 'could_not_open'
instance_status[str(base_name +
str(instance_id))]['status_message'] = str(e)
try:
if sys.platform != 'darwin' and use_xvfb:
print('Closing Xvfb')
vdisplay.stop()
except BaseException as e:
print(e)
return instance_status
try:
driver.get(base_page)
instance_status[
str(base_name +
str(instance_id))]['timestamps']['1_requested_website'] = str(
datetime.utcnow())
# Wait for chat launcher
WebDriverWait(driver, 45).until(
EC.visibility_of_element_located((By.ID, 'nds-chat-launcher')))
# Click on chat launcher
instance_status[str(
base_name +
str(instance_id))]['timestamps']['2_chat_became_available'] = str(
datetime.utcnow())
# print('will open chat')
sign_in_button = driver.find_element_by_id('nds-chat-launcher').click()
# print('flag 0')
# Move to iframe
WebDriverWait(driver, 30).until(
EC.visibility_of_element_located((By.ID, 'nds-chat-iframe')))
iframe = driver.find_element_by_id('nds-chat-iframe')
driver.switch_to_frame(iframe)
# Wait for segmento
# print('flag 1')
WebDriverWait(driver, 30, poll_frequency=0.1).until(
EC.visibility_of_element_located((By.ID, 'nds-chatbot-message-3')))
# print('flag 2')
WebDriverWait(driver, 15, poll_frequency=0.1).until(
EC.text_to_be_present_in_element(
(By.XPATH, '//*[@id="nds-chatbot-message-3"]/div[1]/div'),
'¿A qué segmento perteneces?'))
# print('flag 3')
instance_status[str(base_name + str(
instance_id))]['timestamps']['3_segmento_cliente_available'] = str(
datetime.utcnow())
# Click on segmento
driver.execute_script(
""" (function(e,s){e.src=s;e.onload=function(){jQuery.noConflict();console.log('jQuery 2.2.4 injected');jQuery('#nds-chatbot-message-3 > div.nds-chat-comment-option-wrap > div:nth-child(3)').click()};document.head.appendChild(e);})(document.createElement('script'),'//code.jquery.com/jquery-2.2.4.min.js') """
)
# print('will click segmento')
#clck_s = driver.find_element_by_xpath('//*[@id="nds-chatbot-message-3"]/div[2]/div[3]').click()
# print('Did click segmento')
instance_status[str(base_name + str(
instance_id))]['timestamps']['4_segmento_cliente_selected'] = str(
datetime.utcnow())
time.sleep(10)
# Wait for name question
WebDriverWait(driver, 15, poll_frequency=0.1).until(
EC.visibility_of_element_located((By.ID, 'nds-chatbot-message-4')))
WebDriverWait(driver, 15, poll_frequency=0.1).until(
EC.text_to_be_present_in_element(
(By.XPATH, '//*[@id="nds-chatbot-message-4"]/div/div'),
'¿Cómo te llamas?'))
instance_status[str(base_name + str(
instance_id))]['timestamps']['5_como_te_llamas_prompted'] = str(
datetime.utcnow())
# Get text input field and send name
input_field = driver.find_element_by_id('txMessage')
instance_name = base_name + str(instance_id)
input_field.send_keys(instance_name, Keys.ENTER)
instance_status[
str(base_name +
str(instance_id))]['timestamps']['6_instance_name_sent'] = str(
datetime.utcnow())
# Wait for "¿Qué puedo hacer por ti?" prompt
WebDriverWait(driver, 15, poll_frequency=0.1).until(
EC.visibility_of_element_located((By.ID, 'nds-chatbot-message-5')))
time.sleep(2)
# Send inputs and test responses:
inputs_to_send = {
'por que soy cliente advance':
"HSBC Advance es una propuesta para clientes distinguidos, enfocada a solucionar sus necesidades financieras, ofreciéndole productos y servicios con condiciones preferentes que te ayudarán a lograr tus metas de vida, así como la administración y construcción de tu patrimonio.",
'que significa hsbc':
"HSBC recibe su nombre de Hong Kong and Shanghai Banking Corporation Limited, compañía que se creó en 1865 para financiar el creciente comercio entre Europa, India y China.",
'cual es el telefono de empresarial':
'Para cualquier duda sobre productos o servicios empresariales por favor comunícate a nuestra Línea de Servicios Empresariales:'
} # Finish inputs_to_send
# 'tengo un afore':'HSBC y Principal Afore unen fuerzas para que al término de tu vida laboral tengas la mejor recompensa.',
entered_chat = True
for i_key in inputs_to_send:
input_field.send_keys(i_key, Keys.ENTER)
print('Sent ' + str(i_key))
input_time = datetime.utcnow()
instance_status[str(base_name + str(instance_id))]['timestamps'][
'sent_' + str(i_key.replace(' ', '_'))] = str(input_time)
waiting_for_res = True
time_out = 0
while waiting_for_res:
chat_nds_bubbles = driver.execute_script("""
var z = document.getElementsByClassName('nds-chat-comment-by-nds-chat');
var arr = Array.prototype.slice.call(z);
t = arr.map(function(e){return e.innerText});
return t
""")
for index, el in enumerate(chat_nds_bubbles):
if inputs_to_send[i_key] in el:
print('Got response for ' + str(i_key))
res_time = datetime.utcnow()
instance_status[str(
base_name + str(instance_id))]['timestamps'][
'got_response_for_' +
str(i_key.replace(' ', '_'))] = str(res_time)
waiting_for_res = False
break
time.sleep(1)
time_out += 1
if time_out > 60 * 3:
timed_out_timestamp = datetime.utcnow()
instance_status[str(
base_name + str(instance_id))]['status'] = 'timed_out'
instance_status[str(base_name + str(instance_id))][
'timestamps']['8_timed_out_timestamp'] = str(
timed_out_timestamp)
instance_status[str(
base_name + str(instance_id)
)]['timestamps']['chatbot_history'] = str(chat_nds_bubbles)
break
except BaseException as e:
print('Problems with instance ' + str(instance_id))
print(e)
if 'other_errors' in instance_status[str(base_name + str(
instance_id))]: # Concatenate previous errors if they exists
instance_status[str(base_name +
str(instance_id))]['other_errors'] += str(e)
else:
instance_status[str(base_name +
str(instance_id))]['other_errors'] = str(e)
if 'status' in instance_status[str(base_name + str(instance_id))]:
if instance_status[str(base_name +
str(instance_id))]['status'] != 'timed_out':
instance_status[str(base_name +
str(instance_id))]['status'] += '_correct*'
elif entered_chat:
instance_status[str(base_name +
str(instance_id))]['status'] = 'correct'
else:
instance_status[str(base_name +
str(instance_id))]['status'] = 'timed_out_2'
instance_status[
str(base_name +
str(instance_id))]['timestamps']['9_closing_instance'] = str(
datetime.utcnow())
print('Closing ' + base_name + str(instance_id))
try:
driver.close()
if sys.platform != 'darwin' and use_xvfb:
vdisplay.stop()
except BaseException as e:
instance_status[str(base_name +
str(instance_id))]['closing_error'] = str(e)
return instance_status
def logout(self):
logout_link_elem = WebDriverWait(self.driver, 5).until(
EC.presence_of_element_located((By.ID, "logout"))
)
logout_link_elem.click()
def close_quiz(self):
quit_link_elem = WebDriverWait(self.driver, 5).until(
EC.presence_of_element_located((By.ID, "login_again"))
)
quit_link_elem.click()
def scrapeMatch(profile_url):
global PATH
global MATCH_RANGE
global LIST_RECENT_MATCHES
global INTERNET
try:
driver = webdriver.Chrome(
PATH
) #here we open a browser. I could switch it to headless but I like the look with the browsers open
url = profile_url.replace(
'overview', 'matches'
) #here we do a replace. Codtracker urls make this easy enough
driver.get(url)
for i in range(
1
): #TODO: get rid of this line. Its here because I used to use this line for match range. So maybe the line should stay
element = WebDriverWait(driver, 10).until(
EC.presence_of_element_located((
By.CLASS_NAME,
"match__link"))) #important line. Waits until links appear
ps = driver.find_elements_by_class_name(
'match__link') #then finds the links
element = ps[
i] #we only need the first match for my specific study, but its in this function you may want to make some alterations
#I had trouble generalizing my code to whatever possible study you want to perform
#I'm happy to alter my code, though, if you have a specific study in mind. THe first study I did, the one with the messy
#code, I would have used for i in range(MATCH_RANGE), where MATCH_RANGE = 6. That way, you could get 6, even 20 recent
#matches if you wanted. TODO: fix the MATCH_RANGE thing in the GUI and in this function
match_time = BeautifulSoup(driver.page_source, 'lxml').text.split(
'\n'
) #although the best data comes next, there's some data on this page
#so i grab the current text
driver.execute_script(
"arguments[0].click()", element
) #now we click that first element. Pretty magical selenium
element = WebDriverWait(driver, 10).until(
EC.presence_of_element_located(
(By.CLASS_NAME,
"team__info"))) #now we're off to a recent match
#and we await team__info
content = driver.page_source #grab the html in the match page
team_list = [
] #for each match we create an empty list that will store that match's data
soup = BeautifulSoup(content, 'lxml')
mydivs = soup.findAll("div", {
"class": "team card bordered responsive"
}) #this class works for any game mode, solos, special game modes
for div in mydivs: #the team card is worthless, its the team__info and player__info we want
team_list_row = []
team_info_div = div.find(
"div",
{"class": "team__info"
}) #this is team placement, team stats for that match
team_list_row.append(
team_info_div.text) #we store the team__info
team_players_div = div.findAll(
"div",
{"class": "player__info"}) #now we get the player divs
for player in team_players_div:
team_list_row.append(
player.text
) #and now we add each player to the team_list_row. Each player comes with more match stats
team_list.append(
team_list_row
) #and finally, for each team card, we append to the team_list
LIST_RECENT_MATCHES.append(
[i, profile_url, match_time, team_list, 'root']
) #now we use a global to store our row of data, the match number,the url this stuff came from
#various stats we will finish parsing later, as well as a placeholder 'root'
driver.back(
) #selenium is magic and hits the back button, realizing now that this could probably be deleted since the match range is 1
driver.close(
) #now the driver closes. We could just get the next url in the current driver, so yeah, driver.back() and driver.close() are pointlessly slowing things down
#TODO: check if MATCH_RANGE is 1, if so, dont driver.back(). Also, don't close the driver, get the next url directly from current driver
except:
INTERNET = False #if any of the parsing fails then the internet is checked. It's okay if there was some error because the missing data can be filled
options.add_argument('--disable-dev-shm-usage')
driver = webdriver.Chrome(chrome_options=options)
driver_secondary = webdriver.Chrome(chrome_options=options)
else:
driver = webdriver.Chrome()
driver_secondary = webdriver.Chrome()
driver.get(fino_url)
# click on cookies button
try:
utils.click_cookies_button(driver)
except:
print("Unable to click on accept cookies button")
# Wait 5 seconds for page to load
timeout = 5
try:
WebDriverWait(driver, timeout).until(
EC.visibility_of_element_located((By.CLASS_NAME, "page-numbers")))
except TimeoutException:
print("Timed out waiting for page to load")
driver.quit()
# either use command line input, or get user input
if args.npages is not None:
N_pages_extract = args.npages
else:
xpath_npages = """/html/body[@id='fino_theme_clean']/div[@id='page']/main[@class='main']/div[@class='container']\
/div[@class='row']/div[@class='col-lg-8 col-xs-12']/div[@class='paging group']/a[@class='page-numbers'][3]"""
n_pages = int(
driver.find_element_by_xpath(xpath_npages).text.replace(".", ""))
print("There are {} pages in finofilipino".format(n_pages))
N_pages_extract = input(
"How many pages do you want to extract? (default = {}) ".format(
n_pages))
def process_request(self, request, spider):
PrintFormatUtil.print_line("spider {} : 开始处理 Request".format(spider.name))
if spider.crawl_type.value == 'selenium' and isinstance(request, SeleniumRequest):
self.driver.set_window_size(800, 600)
self.driver.get(request.url)
# copy cookie
for cookie_name, cookie_value in request.cookies.items():
self.driver.add_cookie(
{
'name': cookie_name,
'value': cookie_value
}
)
if request.wait_until:
WebDriverWait(self.driver, request.wait_time).until(request.wait_until)
if request.screen_shot:
# Get the actual page dimensions using javascript
width = self.driver.execute_script(
"return Math.max(document.body.scrollWidth, document.body.offsetWidth, "
"document.documentElement.clientWidth, document.documentElement.scrollWidth, "
"document.documentElement.offsetWidth);")
height = self.driver.execute_script(
"return Math.max(document.body.scrollHeight, document.body.offsetHeight, "
"document.documentElement.clientHeight, document.documentElement.scrollHeight, "
"document.documentElement.offsetHeight);")
# resize
PrintFormatUtil.print_line("reset size {}:{}".format(width, height))
self.driver.set_window_size(width, height)
time.sleep(1)
request.meta['screen_shot'] = self.driver.get_screenshot_as_png()
if request.script:
self.driver.execute_script(request.script)
request.meta['r_dict'] = request.r_dict
body = str.encode(self.driver.page_source)
# Expose the driver via the "meta" attribute
request.meta.update({'driver': self.driver})
return HtmlResponse(
self.driver.current_url,
body=body,
encoding='utf-8',
request=request
)
if spider.crawl_type.value == 'puppeeter' and isinstance(request, PuppeeterRequest):
page = sync(self.driver.newPage())
sync(page.setJavaScriptEnabled(enabled=True))
sync(page.setUserAgent(
'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1'))
sync(page.goto(request.url))
request.meta['r_dict'] = request.r_dict
body = str.encode(sync(page.content()))
# Expose the driver via the "meta" attribute
request.meta.update({'driver': self.driver})
request.meta.update({'page': page})
return HtmlResponse(
request.url,
body=body,
encoding='utf-8',
request=request
)
# Called for each request that goes through the downloader
# middleware.
# Must either:
# - return None: continue processing this request
# - or return a Response object
# - or return a Request object
# - or raise IgnoreRequest: process_exception() methods of
# installed downloader middleware will be called
return None
def WaitForPageLoad(driver):
WebDriverWait(driver, 10).until(lambda d: d.execute_script('return document.readyState') == 'complete')
l = chicun()
x1 = int(l[0] * 0.9)
y1 = int(l[1] * 0.5)
x2 = int(l[0] * 0.1)
driver.swipe(x1, y1, x2, y1, 1000)
def swipeUp():
l = chicun()
x1 = int(l[0] * 0.5)
y1 = int(l[1] * 0.95)
y2 = int(l[1] * 0.35)
driver.swipe(x1, y1, x1, y2, 1000)
WebDriverWait(
driver, 6).until(lambda x: x.find_element_by_id("com.mymoney:id/next_btn"))
for i in range(2):
swipeLeft()
sleep(1)
#点击随手记
driver.find_element_by_id('com.mymoney:id/begin_btn').click()
try:
closBtn = driver.find_element_by_id('com.mymoney:id/close_iv')
except NoSuchElementException:
pass
else:
closBtn.click()
driver.find_element_by_id('com.mymoney:id/nav_setting_btn').click()
def click_toc_from_text_panel(self):
self.driver.find_element_by_css_selector(".readerTextTocBox a").click()
WebDriverWait(self.driver, TEMPER).until(
element_to_be_clickable(
(By.CSS_SELECTOR, ".tocContent > :not(.loadingMessage)")))
def _webdriver_wait(driver):
"""Common WebDriverWait logic with poll frequency."""
return WebDriverWait(driver,
constants.ux.MAX_USER_WAIT_SECONDS,
poll_frequency=constants.ux.POLL_FREQUENCY)
def find_text_filter(self, name):
WebDriverWait(self.driver, TEMPER).until(
element_to_be_clickable(
(By.CSS_SELECTOR, '.textFilter[data-name="{}"]'.format(name))))
return self.driver.find_element_by_css_selector(
'.textFilter[data-name="{}"]'.format(name))
def testWaitUntilNotReturnsIfEvaluatesToFalse(self):
falsum = lambda driver: False
self.assertFalse(WebDriverWait(self.driver, 1).until_not(falsum))
def load_home(self):
self.driver.get(self.base_url + "/?home")
WebDriverWait(self.driver, TEMPER).until(
element_to_be_clickable((By.CSS_SELECTOR, ".header")))
return self
def testWaitUntilNotShouldNotFailIfProduceIgnoredException(self):
ignored = (InvalidElementStateException, StaleElementReferenceException)
self.assertTrue(WebDriverWait(self.driver, 1, 0.7, ignored_exceptions=ignored).until_not(throwSERE))
def load_notifications(self):
self.driver.get(self.base_url + "/notifications")
WebDriverWait(self.driver, TEMPER).until(
element_to_be_clickable(
(By.CSS_SELECTOR, ".notificationsList > .notification")))
return self
def testShouldExplicitlyWaitForASingleElement(self):
self._loadPage("dynamic")
add = self.driver.find_element_by_id("adder")
add.click();
WebDriverWait(self.driver, 3).until(EC.presence_of_element_located((By.ID, "box0"))) # All is well if this doesn't throw.
def load_private_groups(self):
self.driver.get(self.base_url + "/my/groups")
WebDriverWait(self.driver, TEMPER).until(
element_to_be_clickable(
(By.CSS_SELECTOR, ".myGroupsPanel .button")))
return self
def testShouldWaitAtLeastOnce(self):
self._loadPage("simpleTest")
elements_exists = lambda driver: driver.find_elements_by_tag_name('h1')
elements = WebDriverWait(self.driver, 0).until(elements_exists)
self.assertTrue(len(elements) >= 1)
def create_3cxAccount(firstName,
lastName,
emailAddress,
location='Los Angeles, CA - 911',
outbound=False):
chrome_options = Options()
download_directory = "C:\\3cx"
chrome_options.add_experimental_option(
"prefs", {
"download.default_directory": download_directory,
"download.prompt_for_download": False
})
chrome_options.add_argument("--headless")
WINDOW_SIZE = "3840,2160"
chrome_options.add_argument("--window-size=%s" % WINDOW_SIZE)
chrome_options.add_argument("start-maximized")
chrome_options.add_argument("disable-infobars")
chrome_options.add_argument("--disable-extensions")
driver = webdriver.Chrome(options=chrome_options)
driver.command_executor._commands["send_command"] = (
"POST", '/session/$sessionId/chromium/send_command')
params = {
'cmd': 'Page.setDownloadBehavior',
'params': {
'behavior': 'allow',
'downloadPath': download_directory
}
}
command_result = driver.execute("send_command", params)
wait = WebDriverWait(driver, 10)
driver.get("http://3cxdev01/#/login")
wait.until(
ec.visibility_of_element_located((
By.CSS_SELECTOR,
"input[placeholder*='User name or extension number']"))).send_keys(
"") #enter username here
wait.until(
ec.visibility_of_element_located(
(By.CSS_SELECTOR,
"input[placeholder*='Password']"))).send_keys("") #password here
wait.until(
ec.visibility_of_element_located(
(By.CSS_SELECTOR,
"button[class*='btn btn-lg btn-primary btn-block ng-scope']"
))).click()
driver.get("http://3cxdev01/#/app/extensions")
wait.until(ec.visibility_of_element_located((By.ID, "btnExport"))).click()
#Check if export list finish down. Max 10s
if os.path.exists("C:\\3cx\\extensions.csv"):
os.remove("C:\\3cx\\extensions.csv")
now = time.time()
last_time = now + 10
while time.time() <= last_time:
if os.path.exists("C:\\3cx\\extensions.csv"):
break
#find list of available outbound and inbound number
f = pd.read_csv("C:/3cx/extensions.csv")
outboundList = pd.read_csv("exportDID.csv")
keepCol = ['Number']
f = f[keepCol]
keepCol = ['MASK', 'INOFFICE_DEST_NUMBER']
outboundList = outboundList[keepCol]
availableEx = list(f.Number)
availableEx = find_missing(availableEx) #list of available extensions
availableEx = list(map(str, availableEx)) #convert to string
outboundList = outboundList.loc[pd.isnull(outboundList).any(1), :]
outboundList['MASK'].replace({'\*': ''}, regex=True, inplace=True)
outboundList = list(outboundList['MASK'])
outboundList = [i[-4:] for i in outboundList
if '310448' in i] #list of outbound number for 310448XXXX
outboundToUse = set(availableEx).intersection(outboundList)
outboundToUseList = list(outboundToUse)
for x in list(outboundToUse):
availableEx.remove(x)
if (outbound == True):
outboundToUseList.sort()
extensionAdd = str(outboundToUseList[0])
else:
availableEx.sort()
extensionAdd = str(availableEx[0])
print(extensionAdd)
#create extension
wait.until(ec.visibility_of_element_located((By.ID, "btnAdd"))).click()
wait.until(
ec.visibility_of_element_located(
(By.CSS_SELECTOR, "input[placeholder='Extension']"))).clear()
wait.until(
ec.visibility_of_element_located(
(By.CSS_SELECTOR,
"input[placeholder='Extension']"))).send_keys(extensionAdd)
wait.until(
ec.visibility_of_element_located(
(By.CSS_SELECTOR,
"input[placeholder='First Name']"))).send_keys(firstName)
wait.until(
ec.visibility_of_element_located(
(By.CSS_SELECTOR,
"input[placeholder='Last Name']"))).send_keys(lastName)
wait.until(
ec.visibility_of_element_located(
(By.CSS_SELECTOR,
"input[placeholder='Email Address']"))).send_keys(emailAddress)
wait.until(
ec.visibility_of_element_located(
(By.CSS_SELECTOR, "input[placeholder='Outbound Caller ID']"
))).send_keys("310448" + extensionAdd)
wait.until(
ec.visibility_of_element_located(
(By.CSS_SELECTOR, "input[placeholder='ID']"))).clear()
wait.until(
ec.visibility_of_element_located(
(By.CSS_SELECTOR,
"input[placeholder='ID']"))).send_keys(extensionAdd)
wait.until(
ec.visibility_of_element_located(
(By.CSS_SELECTOR, "input[placeholder='Password']"))).clear()
wait.until(
ec.visibility_of_element_located(
(By.CSS_SELECTOR,
"input[placeholder='Password']"))).send_keys("dai123")
element = driver.find_element_by_id("btnSave")
coordinates = element.location_once_scrolled_into_view # returns dict of X, Y coordinates
driver.execute_script('window.scrollTo({}, {});'.format(
coordinates['x'], coordinates['y']))
time.sleep(1)
wait.until(ec.element_to_be_clickable((By.ID, "btnSave"))).click()
wait.until(ec.url_changes(driver.current_url))
if (outbound == True):
#assign inbound rules
driver.get("http://3cxdev01/#/app/inbound_rules")
wait.until(ec.visibility_of_element_located(
(By.ID, "inputSearch"))).send_keys(extensionAdd)
wait.until(
ec.visibility_of_element_located((
By.XPATH,
f"//td[@class='ng-binding' and contains(., '310448{extensionAdd}')]"
))).click()
wait.until(ec.visibility_of_element_located(
(By.ID, "btnEdit"))).click()
wait.until(
ec.visibility_of_element_located(
(By.CSS_SELECTOR,
"input[placeholder='Inbound rule name']"))).clear()
wait.until(
ec.visibility_of_element_located(
(By.CSS_SELECTOR, "input[placeholder='Inbound rule name']"
))).send_keys(firstName + " " + lastName)
Select(
wait.until(
ec.visibility_of_element_located((
By.CSS_SELECTOR,
"select[class='form-control ng-pristine ng-untouched ng-valid ng-not-empty']"
)))).select_by_visible_text('Extension')
wait.until(
ec.visibility_of_element_located(
(By.XPATH,
"//span[@class='ng-binding ng-scope' and contains(., '0000')]"
))).click()
wait.until(
ec.visibility_of_element_located((
By.XPATH,
"//*[@id=\"app\"]/div/div[3]/div[2]/div/div[2]/inbound-rule-editor-contents/panel[2]/div/div[2]/ng-transclude/routing-control/div/div[3]/select-control/div/div/div/input[1]"
))).send_keys(extensionAdd)
wait.until(
ec.visibility_of_element_located((
By.XPATH,
f"//span[@class='ng-binding ng-scope' and contains(., '{extensionAdd}')]"
))).click()
Select(
wait.until(
ec.visibility_of_element_located((
By.XPATH,
"//div[@class='form-group' and contains(., 'Destination for calls outside office hours')]/select"
)))).select_by_visible_text('Extension')
wait.until(
ec.visibility_of_element_located(
(By.XPATH,
"//span[@class='ng-binding ng-scope' and contains(., '0000')]"
))).click()
wait.until(
ec.visibility_of_element_located((
By.XPATH,
"//*[@id=\"app\"]/div/div[3]/div[2]/div/div[2]/inbound-rule-editor-contents/panel[2]/div/div[2]/ng-transclude/routing-control/div/div[8]/select-control/div/div/div/input[1]"
))).send_keys(extensionAdd)
driver.implicitly_wait(5)
time.sleep(3)
driver.find_elements_by_xpath(
f"//span[@class='ng-binding ng-scope' and contains(., '{extensionAdd}')]"
)[1].click()
wait.until(ec.visibility_of_element_located(
(By.ID, "btnSave"))).click()
wait.until(ec.url_changes(driver.current_url))
#Add to correct 911 group
driver.get("http://3cxdev01/#/app/groups")
wait.until(
ec.visibility_of_element_located(
(By.XPATH,
f"//td[@class='ng-binding' and contains(., '{location}')]"
))).click()
wait.until(ec.visibility_of_element_located((By.ID, "btnEdit"))).click()
wait.until(ec.url_changes(driver.current_url))
wait.until(ec.visibility_of_element_located((By.ID, "btnAdd"))).click()
wait.until(ec.visibility_of_element_located(
(By.ID, "inputSearch"))).send_keys(extensionAdd)
wait.until(
ec.visibility_of_element_located((
By.XPATH,
f"//td[@class='checkbox-row ng-binding' and contains(., {extensionAdd})]"
))).click()
#timing issue, use sleep for now
time.sleep(4)
driver.find_element_by_xpath(
"/html/body/div[1]/div/div/div[3]/button[1]").click()
time.sleep(4)
driver.find_element_by_id("btnSave").click()
wait.until(ec.url_changes(driver.current_url))
return driver, extensionAdd
abc = pd.read_excel('C:\\Users\\acer\\Downloads\\cds\\mat.xls', header=None, index_col=False)
f = open('data.csv', 'a')
var=0
continueCheck = False
for item in abc.index:
print(abc[0][item])
website = str(abc[0][item])
if website == '6annonce.com':
continueCheck = True
continue
if continueCheck:
driver.get('https://pro.similarweb.com/#/website/worldwide-overview/'+website+'/*/999/3m?webSource=Total')
# try:
if driver.title != 'Pardon Our Interruption':
wait = WebDriverWait(driver, 40)
my_div = wait.until(ec.visibility_of_element_located((By.CLASS_NAME, "wwo-web-ranks")))
else:
my_div = checkTitleAgain(driver)
# except:
# my_div = ''
try:
divs = my_div.find_elements_by_class_name('ranks-row')
category = divs[2].text
category = category.split('\n')
cat_name = category[1]
print(cat_name)
f.write(website + '--->>>' + cat_name)
f.write('\n')
except:
