def staff_directory(self): HR = WebDriverWait(self.driver, 40).until( EC.element_to_be_clickable( (By.LINK_TEXT, Dlocator.HR_sidemenu_linktext))) action = ActionChains(self.driver) action.move_to_element(HR).perform() self.driver.implicitly_wait(7) Staff_directory = WebDriverWait(self.driver, 60).until( EC.element_to_be_clickable( (By.LINK_TEXT, Dlocator.Staffdirectory_sidemenu_linktext))) Staff_directory.click()
def step_impl(context, Username, password): WebDriverWait(context.driver, 10).until(EC.element_to_be_clickable((By.ID, 'i0116'))) Log.Search_Username_Textbox(context).clear() Log.Search_Username_Textbox(context).send_keys(Username) Log.Search_Nextstep_Button(context).click() WebDriverWait(context.driver, 10).until(EC.element_to_be_clickable((By.ID, 'i0118'))) Log.Search_Password_Textbox(context).clear() Log.Search_Password_Textbox(context).send_keys(password) Log.Search_Signin_Button(context).click()
def checkZipcodeValidation(self, zipcode=None): element_present = EC.element_to_be_clickable((By.ID, "zip-code-popup-opener")) WebDriverWait(self.driver, 10).until(element_present) self.driver.find_element_by_id("zip-code-popup-opener").click() time.sleep(3) self.driver.find_element_by_id("zipcode").send_keys(zipcode) self.driver.find_element_by_id("zipcode").send_keys(Keys.ENTER)
def index_page(page): """ 抓取索引页 :param page: 页码 """ print('正在抓取第', page, '页') try: url = 'https://s.taobao.com/search?q=' + quote(KEYWORD) print(url) browser.get(url) if page > 1: input = wait.until( EC.presence_of_element_located( (By.CSS_SELECTOR, '#mainsrp-pager div.form > input'))) submit = wait.until( EC.element_to_be_clickable( (By.CSS_SELECTOR, '#mainsrp-pager div.form > span.btn.J_Submit'))) input.clear() input.send_keys(page) submit.click() wait.until( EC.text_to_be_present_in_element( (By.CSS_SELECTOR, '#mainsrp-pager li.item.active > span'), str(page))) wait.until( EC.presence_of_element_located( (By.CSS_SELECTOR, '.m-itemlist .items .item'))) get_products() except TimeoutError: index_page(page)
def should_be_successful_message(self): page = MainPage(self.browser, link) self.add_product_to_bag() WebDriverWait(self.browser, 5).until( EC.element_to_be_clickable(*ProductPageLocators.SUCCESSFUL_MESSAGE)) assert page.is_element_present( *ProductPageLocators.SUCCESSFUL_MESSAGE),\ "There is not a message about successful editing a product"
def remove_popup_banner(self): try: self.driver.implicitly_wait(10) WebDriverWait(self.driver, 20).until( EC.element_to_be_clickable( (By.CSS_SELECTOR, 'a.next-dialog-close'))).click() print('==> Clicked and remove popup banner') self.driver.implicitly_wait(5) self.driver.execute_script("window.scrollTo(0, 200)") sleep(5) except Exception as e: print('Error in clicking popup banner BTN : ' + str(e))
def testLogOut(self): switch = False self.username.send_keys('TestUser') self.password.send_keys('superhemligt') self.submit.send_keys(Keys.RETURN) self.driver.implicitly_wait(3) self.assertIn('Min sida', self.driver.page_source) self.log_out = WebDriverWait(self.driver, 20).until( EC.element_to_be_clickable((By.CLASS_NAME, 'sign-out'))) self.log_out.click() if WebDriverWait(self.driver, 20).until(EC.url_changes): switch = True self.assertIs(switch, True)
def get_following_names(self, profilename, By=None): url = "https://www.hedieh.com/"+profilename+"/" self.driver.get(url) time.sleep(3) path = "/html/" wait = WebDriverWait(self.driver, 10) following_button = wait.until(EC.element_to_be_clickable((By.XPATH, path))) following_button.click() names_following = self.get_names(self.driver, profilename) f = open("following_names.txt", "w") for i in names_following: f.write(i) f.write("\n") return names_following
def get_component(self, metodo, valor): global EC from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.support.ui import WebDriverWait wait = WebDriverWait(self.driver, 5) try: element = wait.until(EC.element_to_be_clickable((metodo, valor))) return element except: time.sleep(2) try: self.component = self.driver.find_element(metodo, valor) return self.component except: #print "Busca por elemento excedeu o tempo maximo.",valor return None
def unfollow_with_file(self, filename, By=None): f = open(filename, "r") lines = f.readlines() not_following_back = [] for i in lines: not_following_back.append(i[:-1]) wait = WebDriverWait(self.driver, 10) for i in not_following_back: url = "https://www.hedieh.com/"+i+"/" self.driver.get(url) unfollow1 = wait.until( EC.presence_of_all_elements_located((By.CSS_SELECTOR, 'button'))) unfollow1[1].click() time.sleep(4) path = "/html/" unfollow2 = wait.until( EC.element_to_be_clickable((By.XPATH, path))) unfollow2.click() time.sleep(randint(5,13))
def waitForElement(self, locator, locatorType="id", timeout=10, pollFrequency=0.5): element = None try: byType = self.getByType(locatorType) self.log.info("Waiting for maximum :: " + str(timeout) + " :: seconds for element to be clickable") wait = WebDriverWait(self.driver, timeout=timeout, poll_frequency=pollFrequency, ignored_exceptions=[ NoSuchElementException, ElementNotVisibleException, ElementNotSelectableException ]) element = wait.until(EC.element_to_be_clickable((byType, locator))) self.log.info("Element appeared on the web page") except: self.log.info("Element not appeared on the web page") print_stack() return element
def waitHomeButton(self): element_present = EC.element_to_be_clickable((By.LINK_TEXT, "Home")) WebDriverWait(self.driver, 100).until(element_present)
def clickSendButton(self): wait = WebDriverWait(self.driver, 50) sendButton = wait.until( EC.element_to_be_clickable( (By.CSS_SELECTOR, "[data-test-id='resendBtn']"))) sendButton.click()
def task_1_add_product(self, driver): self.wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, OKULARY))) self.driver.find_element_by_css_selector(OKULARY).clear() self.driver.find_element_by_css_selector(OKULARY).send_keys(3) self.driver.find_element_by_css_selector(ADD_BUTTON) return self.driver
def clickChangeLocation(self): element_present = EC.element_to_be_clickable((By.ID, "zip-code-popup-opener")) WebDriverWait(self.driver, 150).until(element_present) self.driver.find_element_by_id("zip-code-popup-opener").click()
def logOut(self): element_present = EC.element_to_be_clickable( (By.ID, "ctl00_uxLoginStatus")) WebDriverWait(self.driver, 20).until(element_present) self.driver.find_element_by_link_text("LOGOUT").click()
def clickOnElement(self, locator=None): wait = WebDriverWait(self.driver, 100) element = wait.until( EC.element_to_be_clickable((By.CSS_SELECTOR, locator))) element.click()
def get_element( self, xpath ): #this method is the provider of waits still load any elements return self.__driver_wait.until( EC.element_to_be_clickable(By.XPATH, xpath))
def click(self, locator): wait = WebDriverWait(self.driver, 15) clickCategory = wait.until( EC.element_to_be_clickable((By.CSS_SELECTOR, locator))) clickCategory.click()
def scroll_down(self): global position try: wait = WebDriverWait(self.driver, 10) wait.until( EC.element_to_be_clickable( (By.CSS_SELECTOR, '#form-cheapest-acc-hot13445 > button:nth-child(1)'))) except TimeoutException: print("Loading took too much time!") try: print('Scrolling page ...') # self.driver.execute_script('window.scrollTo(0, document.body.scrollHeight);') read_mores = self.driver.find_elements_by_xpath('//form/button') for read_more in read_mores: self.driver.execute_script("arguments[0].scrollIntoView();", read_more) # read_more.click() sleep(1) # even 25 seconds may not be enough to load the page! # todo AJAX: maximum 30 seconds print('Waiting ajax full load ...') sleep(11) # wait ajax full load print("Scraping page ...") soup = BeautifulSoup(self.driver.page_source, 'lxml') hotel_list = soup.find_all( 'article', {'class': 'crosselling-line availability-item'}) euro_symbol = '€' print("\n\tSnapshoot:\n") try: for i, hotel in enumerate(hotel_list): self.all_positions.append(i + 1) hotel_name = hotel.find('a', { 'data-tl': 'acc-title' }).getText() # fixme: remove whitespaces REF: https://stackoverrun.com/es/q/743639 hotel_name = ' '.join(hotel_name.split()) self.all_hotels.append(hotel_name) # print(" %d - %s" % (i + 1, hotel_name)) hotel_zone = hotel.find('span', { 'class': '_hotelzone' }).getText().strip(',') hotel_name = ' '.join(hotel_name.split()) self.all_zones.append(hotel_zone) hotel_integer = hotel.find('span', { 'class': 'hotel-price' }).getText() hotel_decimal = hotel.find('span', { 'class': 'hotel-price-decimal' }).getText().strip('€') hotel_integer = hotel_integer.replace(',', '') hotel_price = '{}.{}'.format(hotel_integer, hotel_decimal) hotel_price = float(hotel_price) hotel_price = "{0:.2f}".format(hotel_price) self.all_prices.append(hotel_price) if len(hotel_price) == 5: hotel_price = " " + hotel_price if len(hotel_price) == 6: hotel_price = " " + hotel_price if len(hotel_price) == 7: hotel_price = " " + hotel_price if len(hotel_price) == 8: hotel_price = "" + hotel_price if i < 9: print(" %d - %s %s %s - %s" % (i + 1, hotel_price, euro_symbol, hotel_name, hotel_zone)) else: print("%d - %s %s %s - %s" % (i + 1, hotel_price, euro_symbol, hotel_name, hotel_zone)) print("\n\tRanking:\n") # float cast new_prices = [] for element in self.all_prices: rank = float(element) new_prices.append(rank) # final list display_list = list( zip(self.all_positions, self.all_hotels, new_prices, self.all_zones)) ranking = sorted(display_list, key=operator.itemgetter(2)) # todo REF: https://discuss.codecademy.com/t/how-can-i-sort-a-zipped-object/454412/6 for j, k, v, w in ranking: if v < 100.00: print(" ", "{0:.2f}".format(v), k) if 99.00 < v < 1000.00: print(" ", "{0:.2f}".format(v), k) if 999.00 < v < 10000.00: print(" ", "{0:.2f}".format(v), k) if v > 9999.00: print("", "{0:.2f}".format(v), k) self.cheap = ranking[0] self.data = ranking print('\nCheapest reservation: ', self.cheap[1], self.cheap[2], euro_symbol) # self.display = display_list[7] # print('Target button number: ', self.display.index(self.cheap[0])) self.display = display_list for i, collation in enumerate(display_list): if collation[1] == self.cheap[1]: position = i print('Pointing to the target button ', position + 1, ' ...') self.index = str(position) if self.error is False: self.target_button(self.index) sleep(2) except Exception as e: self.error = True print(e) print( 'Some error occurred while trying to scratch the hotel list' ) # activate page analyzer if mode == "d": print('Downloading html to computer ...') file = open("bedsonline.html", "w") file.write(soup.prettify()) file.close() except NoSuchElementException: print('Some error occurred while trying to scroll down') self.error = True
def search_target_profile(self): try: # todo: filling the search bar # search_bar.send_keys(self.target_username).click # this line is wrong, read below please # fixme: you cannot enter text directly use the autocomplete square icon to the right of the field WebDriverWait(self.driver, 100).until( EC.visibility_of_element_located( (By.XPATH, '//section/div/form/fieldset[1]/div/a/span'))).click() # todo: https://dzone.com/articles/perform-actions-using-javascript-in-python-seleniu # Method 1: Executing JavaScript at Document Root Level # javaScript = "document.getElementsByClassName('ui-dialog-titlebar-close ui-corner-all')[0].click();" # self.driver.execute_script(javaScript) # Method 2: Executing JavaScript at Element Level: print("Manipulating search engine ...") try: wait = WebDriverWait(self.driver, 100) wait.until( EC.element_to_be_clickable( (By.CSS_SELECTOR, "#continent-picker-tab > li:nth-child" "(" + self.target_continent + ") > a:nth-child(1)"))) except TimeoutException: print("Searching took too much time!") if self.target_continent != '4': picker = WebDriverWait(self.driver, 100).until( EC.visibility_of_element_located( (By.CSS_SELECTOR, "#continent-picker-tab > li:nth-child(" + self.target_continent + ") > a:nth-child(1)"))) self.driver.execute_script("arguments[0].click();", picker) sleep(1) picker = WebDriverWait(self.driver, 100).until( EC.visibility_of_element_located( (By.XPATH, "//section/article[1]/div/ul[" + self.target_country_col + "]/li[" + self.target_country_row + "]/a"))) self.driver.execute_script("arguments[0].click();", picker) sleep(1) picker = WebDriverWait(self.driver, 100).until( EC.visibility_of_element_located( (By.XPATH, "//section/article[2]/div/ul[" + self.target_city_col + "]/li[" + self.target_city_row + "]/a"))) self.driver.execute_script("arguments[0].click();", picker) sleep(1) picker = WebDriverWait(self.driver, 100).until( EC.visibility_of_element_located( (By.XPATH, "//article[3]/div/ul[1]/li[1]/a"))) self.driver.execute_script("arguments[0].click();", picker) sleep(1) # search button login_button = WebDriverWait(self.driver, 100).until( EC.visibility_of_element_located( (By.XPATH, '//*[@id="mainsearch"]'))) print('Loading page ...') login_button.submit() except Exception: self.error = True print('Could not find search bar')
).click() time.sleep(4) driver.find_element_by_xpath("//*[contains(text(), 'Choice2')]").click() time.sleep(4) print("Select checkbox Multiple choice") driver.find_element(By.ID, "checkbox-00000018-3").click() time.sleep(2) driver.find_element( By.XPATH, "//body/form[@id='form']/div/div/div[9]/div[1]/div[4]/div[1]/input[1]" ).send_keys("testing 1") # print("Verification") driver.execute_script( "arguments[0].click();", WebDriverWait(driver, 10).until( EC.element_to_be_clickable(( By.XPATH, "//span[@class='recaptcha-checkbox goog-inline-block recaptcha-checkbox-unchecked rc-anchor-checkbox']/div[@class='recaptcha-checkbox-checkmark']" )))) time.sleep(4) print("Submitting form") driver.find_element( By.XPATH, "/html[1]/body[1]/form[1]/div[1]/div[3]/div[2]/button[1]").click() time.sleep(3) # closing browser tab driver.close()
def waitElement1(self, locator=None): element_present = EC.element_to_be_clickable( (By.CSS_SELECTOR, locator)) WebDriverWait(self.driver, 100).until(element_present)
def Copy(self): time.sleep(10) copy = WebDriverWait(self.browser, 10).until( EC.element_to_be_clickable(self.Locator_login_buttons["Copy"])) copy.click()
def step_impl(context): HomePage.Open(context) WebDriverWait(context.driver, 10).until(EC.element_to_be_clickable((By.ID, "i0116"))) logpage.Signin_Go(context).click
def linkdin(name): browser = webdriver.Chrome('chromedriver.exe') wait = WebDriverWait(browser, 30) browser.get('https://www.linkedin.com/uas/login') username = "******" password = "******" search_item = name elementID = browser.find_element_by_id('username') elementID.send_keys(username) elementID = browser.find_element_by_id('password') elementID.send_keys(password) elementID.submit() wait.until(EC.element_to_be_clickable((By.ID, "nav-typeahead-wormhole"))) browser.find_element_by_css_selector("div.nav-search-typeahead input").send_keys(search_item) browser.find_element_by_css_selector("div.nav-search-typeahead input").send_keys(Keys.ENTER) profilesQueued = [] wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, "span.name.actor-name"))) wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, "h3.search-results__total"))) # Pour ne pas avoir aucun resultats scheight = .1 while scheight < 1.0: browser.execute_script("window.scrollTo(0, document.body.scrollHeight*%s);" % scheight) scheight += .1 try: print(browser.find_element_by_css_selector('div.search-result__wrapper').text) except(NoSuchElementException): print('No div.div.search-result__wrapper') finally: pass profils = browser.find_elements_by_css_selector("div.search-result__wrapper") name_location = profils[0].find_element_by_css_selector("span.name.actor-name") browser.execute_script("arguments[0].scrollIntoView({behavior: 'smooth', block: 'center', inline: 'nearest'});", name_location) name = name_location.text.encode('utf-8') #name = name.lower() name = name.decode() if search_item in name: url_profil = profils[0].find_element_by_css_selector("a.search-result__result-link").get_attribute('href') profilesQueued.append(url_profil) infoo = { "name_linkdin": "None", "profile_title_linkdin": "None", "loc": "None", "information": "None" } educationn = { "college_name": "None", "degree_name": "None", "degree_year": "None" } experiencee = { "company_name": "None", "job_title": "None", "joining_date": "None", "exp": "None" } linkdin_dict = { "info": "None", "education": "None", "experience": "None", "competences": "None" } if (len(profilesQueued) == 1): fullLink = profilesQueued.pop() browser.get(fullLink) SCROLL_PAUSE_TIME = 5 last_height = browser.execute_script("return document.body.scrollHeight") # Get scroll height for i in range(3): browser.execute_script("window.scrollTo(0, document.body.scrollHeight);") time.sleep(SCROLL_PAUSE_TIME) new_height = browser.execute_script("return document.body.scrollHeight") if new_height == last_height: break last_height = new_height ## Info soup = BeautifulSoup(browser.page_source, "html.parser") name_div = soup.find('div', {'class': 'flex-1 mr5'}) if name_div is not None: name_loc = name_div.find_all('ul') if name_loc is not None: name = name_loc[0].find('li').get_text().strip() loc = name_loc[1].find('li').get_text().strip() infoo["name_linkdin"] = name infoo["loc"] = loc else: name_loc = 'None' profile_title = name_div.find('h2').get_text().strip() infoo["profile_title_linkdin"] = profile_title else: name_div = 'None' ##Information name_information = soup.find('div', {'class': 'pv-oc ember-view'}) if name_information is not None: content_information = name_information.find('p') if content_information is not None: content_information = content_information.get_text().strip() content_information = content_information.replace('\n','') content_information = content_information.replace('voir plus','') infoo["information"] = content_information else: content_information = 'None' infoo["information"] = content_information linkdin_dict["info"] = infoo ## Experience # testExp = exp_section.get_text().strip() # info.append(testExp) try: exp_section = soup.find('section', {'id': 'experience-section'}) try: exp_section = exp_section.find('ul') try: li_tags = exp_section.find('div') try: a_tags = li_tags.find('a') try: job_title = a_tags.find('h3').get_text().strip() except (NoSuchElementException, WebDriverException, StaleElementReferenceException): job_title = 'None' finally: pass experiencee["job_title"] = job_title try: company_name = a_tags.find_all('p')[1].get_text().strip() except (NoSuchElementException, WebDriverException, StaleElementReferenceException): company_name = 'None' finally: pass experiencee["company_name"] = company_name try: joining_date = a_tags.find_all('h4')[0].find_all('span')[1].get_text().strip() except (NoSuchElementException, WebDriverException, StaleElementReferenceException): joining_date ='None' finally: pass experiencee["joining_date"] = joining_date try: exp = a_tags.find_all('h4')[1].find_all('span')[1].get_text().strip() except (NoSuchElementException, WebDriverException, StaleElementReferenceException): exp = 'None' finally: pass experiencee["exp"] = exp except (NoSuchElementException, WebDriverException, StaleElementReferenceException): a_tags = 'None' finally: pass except (NoSuchElementException, WebDriverException, StaleElementReferenceException): li_tags = 'None' finally: pass except (AttributeError, NoSuchElementException, WebDriverException, StaleElementReferenceException): exp_section = 'None' finally: pass except (AttributeError, NoSuchElementException, WebDriverException, StaleElementReferenceException): exp_section = 'None' finally: pass linkdin_dict["experience"] = experiencee ## Formation browser.page_source scheight = .1 while scheight < 1.0: browser.execute_script("window.scrollTo(0, document.body.scrollHeight*%s);" % scheight) scheight += .1 try: wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, 'section#education-section'))) try: edu_section = browser.find_element_by_css_selector("section#education-section") # testExp = edu_section.text # info.append(testExp) try: college_name = edu_section.find_element_by_css_selector('h3.pv-entity__school-name.t-16.t-black.t-bold').text except (NoSuchElementException, WebDriverException, StaleElementReferenceException): college_name = 'None' finally: pass educationn["college_name"] = college_name try: degree_name = edu_section.find_element_by_css_selector('p.pv-entity__secondary-title.pv-entity__degree-name.t-14.t-black.t-normal').text except (NoSuchElementException, WebDriverException, StaleElementReferenceException): degree_name = 'None' finally: pass educationn["degree_name"] = degree_name try: degree_year = edu_section.find_element_by_css_selector('p.pv-entity__dates.t-14.t-black--light.t-normal') degree_year = degree_year.find_element_by_css_selector('span:nth-child(2)').text except (NoSuchElementException, WebDriverException, StaleElementReferenceException): degree_year = 'None' finally: pass educationn["degree_year"] = degree_year except (NoSuchElementException, WebDriverException, StaleElementReferenceException): edu_section = 'None' finally: pass except: print('No education') finally: pass linkdin_dict["education"] = educationn # Competences Professionnelles browser.page_source scheight = .1 while scheight < 1.0: browser.execute_script("window.scrollTo(0, document.body.scrollHeight*%s);" % scheight) scheight += .1 try: wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, 'section.pv-profile-section.pv-skill-categories-section.artdeco-container-card.ember-view'))) try: competence_section = browser.find_element_by_css_selector('section.pv-profile-section.pv-skill-categories-section.artdeco-container-card.ember-view') button_skills = competence_section.find_element_by_css_selector('button.pv-profile-section__card-action-bar.pv-skills-section__additional-skills.artdeco-container-card-action-bar.artdeco-button.artdeco-button--tertiary.artdeco-button--3.artdeco-button--fluid') button_skills.send_keys(Keys.ENTER) competence_section = browser.find_element_by_css_selector('section.pv-profile-section.pv-skill-categories-section.artdeco-container-card.ember-view') try: competence_section_1 = competence_section.find_element_by_css_selector('ol.pv-skill-categories-section__top-skills.pv-profile-section__section-info.section-info.pb1') competence_section_1 = competence_section_1.text.split(",") except: competence_section_1 = 'None' finally: pass try: competence_section_2 = competence_section.find_element_by_css_selector('div#skill-categories-expanded') competence_div = competence_section_2.find_element_by_css_selector('div.pv-skill-category-list.pv-profile-section__section-info.mb6.ember-view') competence_ol = competence_div.find_element_by_css_selector('ol.pv-skill-category-list__skills_list.list-style-none') competence_ol = competence_ol.text.split(",") competence_section_1 = competence_ol + competence_section_1 except: competence_section_2 = 'None' finally: pass except: competence_section = 'None' competence_section_1 = 'None' finally: pass except: print('No skills') competence_section_1 = 'None' finally: pass linkdin_dict["competences"] = competence_section_1 else: message='No profil fineded' print(linkdin_dict) browser.close() return linkdin_dict
def task_1(self, driver): self.wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, ZAD_1))) self.driver.find_element_by_css_selector(ZAD_1).click() return self.driver
def waitChangeLocation(self): element_present = EC.element_to_be_clickable((By.ID, "zip-code-popup-opener")) WebDriverWait(self.driver, 150).until(element_present)
def esperar_elemento(self, locator, MyTextElement=None): Get_Entity = Functions.get_entity(self, locator) if Get_Entity is None: return print("No se encontro el Valor en el Json definido") else: try: if self.json_GetFieldBy.lower() == "id": wait = WebDriverWait(self.driver, 20) #TIEMPO DE ESPERA wait.until( EC.visibility_of_element_located( (By.ID, self.json_ValueToFind))) wait.until( EC.element_to_be_clickable( (By.ID, self.json_ValueToFind))) print(u"Esperar_Elemento: Se visualizo el elemento " + locator) return True if self.json_GetFieldBy.lower() == "name": wait = WebDriverWait(self.driver, 20) wait.until( EC.isibility_of_element_located( (By.NAME, self.json_ValueToFind))) wait.until( EC.isibility_of_element_located( (By.NAME, self.json_ValueToFind))) print(u"Esperar_Elemento: Se visualizo el elemento " + locator) return True if self.json_GetFieldBy.lower() == "xpath": wait = WebDriverWait(self.driver, 20) if MyTextElement is not None: self.json_ValueToFind = self.json_ValueToFind.format( MyTextElement) print(self.json_ValueToFind) wait.until( EC.visibility_of_element_located( (By.XPATH, self.json_ValueToFind))) wait.until( EC.element_to_be_clickable( (By.XPATH, self.json_ValueToFind))) print(u"Esperar_Elemento: Se visualizo el elemento " + locator) return True if self.json_GetFieldBy.lower() == "link": wait = WebDriverWait(self.driver, 20) wait.until( EC.visibility_of_element_located( (By.LINK_TEXT, self.json_ValueToFind))) wait.until( EC.EC.element_to_be_clickable( (By.PARTIAL_LINK_TEXT, self.json_ValueToFind))) print(u"Esperar_Elemento: Se visualizo el elemento " + locator) return True except TimeoutError: print(u"Esperar_Elemento: No presente " + locator) Functions.tearDow(self) except NoSuchElementException(self): print(u"Esperar_Elemento: No presente " + locator)
def facebook(name): username='******' password='******' url='https://www.facebook.com/' try: chrome_options = webdriver.ChromeOptions() prefs = {"profile.default_content_setting_values.notifications" : 2} chrome_options.add_experimental_option("prefs",prefs) driver = webdriver.Chrome(chrome_options=chrome_options) #action = webdriver.ActionChains(driver) #pour vider les fichiers--------------------------------------------------------------------------- mon_fichier = open("fichier.txt", "w", encoding="utf-8") mon_fichier.write("") mon_fichier.close() facebook_dict = { "photos": "None", "videos": "None", "sports": "None", "music": "None", "books": "None", "likes": "None" } try: driver.get(url) #etape d'authentification--------------------------------------------------------------------------- #---------------------------------------------------------------------------------------------------- driver.find_element_by_id('email').send_keys(username) driver.find_element_by_id('pass').send_keys(password) driver.find_element_by_id('loginbutton').click() except(NoSuchElementException, WebDriverException, StaleElementReferenceException): driver.close() chrome_options = webdriver.ChromeOptions() prefs = {"profile.default_content_setting_values.notifications" : 2} chrome_options.add_experimental_option("prefs",prefs) driver = webdriver.Chrome(chrome_options=chrome_options) driver.get(url) #etape d'authentification--------------------------------------------------------------------------- #---------------------------------------------------------------------------------------------------- driver.find_element_by_id('email').send_keys(username) driver.find_element_by_id('pass').send_keys(password) driver.find_element_by_id('loginbutton').click() finally: pass #etape de recherche--------------------------------------------------------------------------- #---------------------------------------------------------------------------------------------------- wait = WebDriverWait(driver, 30) wait.until( EC.presence_of_element_located((By.XPATH,"//input[@data-testid='search_input' or @name='q']"))) element=driver.find_element_by_xpath("//input[@data-testid='search_input' or @name='q']") element.send_keys('bassma maachi') driver.find_element_by_xpath("//button[@data-testid='facebar_search_button']").click() wait.until( EC.element_to_be_clickable((By.XPATH,"//a[@data-testid='see_more_header-ENTITY_USER-user']"))) driver.find_element_by_xpath("//a[@data-testid='see_more_header-ENTITY_USER-user']").click() #etape de cibler les resultats--------------------------------------------------------------------------- #---------------------------------------------------------------------------------------------------- #wait.until( EC.presence_of_element_located((By.XPATH,"//div[@class='clearfix pbm']"))) element_from=driver.find_element_by_xpath("//div[@class='clearfix pbm']") wait.until( EC.presence_of_element_located((By.XPATH,"//div[@class='_1qkq _1qkx']"))) element_from1=element_from.find_element_by_xpath("//div[@class='_1qkq _1qkx']") wait.until( EC.presence_of_element_located((By.XPATH,"//div[@class='_1yt' or @id='BrowseResultsContainer']"))) element_from2=element_from1.find_element_by_xpath("//div[@class='_1yt' or @id='BrowseResultsContainer']") wait.until( EC.presence_of_element_located((By.XPATH,"//div[@class='_4p2o _87m1']"))) try: wait.until( EC.presence_of_element_located((By.XPATH,"//div[@class='_4p2o _87m1']"))) wait.until( EC.presence_of_element_located((By.XPATH,"//div[@class='_4p2o _87m1']"))) list=element_from2.find_elements_by_xpath("//div[@class='_4p2o _87m1']") except(StaleElementReferenceException): driver.get(driver.current_url) list=element_from2.find_elements_by_xpath("//div[@class='_4p2o _87m1']") finally: pass #etape de stockage--------------------------------------------------------------------------- #---------------------------------------------------------------------------------------------------- mon_fichier = open("fichier.txt", "a", encoding="utf-8") for i in range(len(list)): element_from3=list[i] WebDriverWait(driver, 150).until( EC.presence_of_element_located((By.TAG_NAME,"span"))) spane=(element_from3.find_elements_by_tag_name('span'))[0] s_text=spane.text mon_fichier.write(s_text+"\n") WebDriverWait(driver, 150).until( EC.presence_of_element_located((By.TAG_NAME,"img"))) image=((element_from3.find_elements_by_tag_name('img'))[0]).get_attribute("src") mon_fichier.write(image+"\n") mon_fichier.close() #etape :choix de profil--------------------------------------------------------------------------- #---------------------------------------------------------------------------------------------------- profile=list[0] profile_elem=profile.find_element_by_xpath("//a[@class='_32mo']") profile_elem.click() print(driver.current_url) WebDriverWait(driver, 400).until( EC.presence_of_element_located((By.XPATH,"//div[@class='clearfix _ikh _3-8y']"))) element=driver.find_element_by_xpath("//div[@class='clearfix _ikh _3-8y']") element1=element.find_element_by_xpath("//div[@class='_3-96']") profile_barre=element1.find_element_by_xpath("//ul[@class='uiList _3-8x _2pic _4kg' or @id='u_0_1n']") barre_infos=profile_barre.find_elements_by_tag_name('li') print(len(barre_infos)) #etape :stockage dans le fichier--------------------------------------------------------------------------- #---------------------------------------------------------------------------------------------------- file_infos = open("scrapping.csv", "w", encoding='utf-8') try: writer = csv.writer(file_infos) writer.writerow( ['information-----------------------'] ) for i in range(len(barre_infos)): writer.writerow( [barre_infos[i].text] ) print(barre_infos[i].text) writer.writerow( ['----------------------------------'] ) finally: file_infos.close() #etape :about page--------------------------------------------------------------------------- #---------------------------------------------------------------------------------------------------- profile_barre=driver.find_element_by_xpath("//div[@class='clearfix' or @id='fbTimelineHeadline']") profile_ul=profile_barre.find_element_by_xpath("//ul[@class='_6_7 clearfix' or @id='u_0_12']") list=profile_ul.find_elements_by_tag_name('li') barre_infos=list[1].find_element_by_tag_name('a') barre_infos.click() print(driver.current_url) #on est dans la page about-----------------------------on va cibler la partir 'mention de j'aime'----------------- WebDriverWait(driver, 400).until( EC.presence_of_element_located((By.XPATH,"//div[@class='fb_content clearfix' or @id='content']"))) body=driver.find_element_by_xpath("//div[@class='fb_content clearfix' or @id='content']") bodyA=body.find_element_by_xpath("//div[@id='mainContainer']") bodyB=bodyA.find_element_by_xpath("//div[@class='clearfix' or @id='contentCol']") html = driver.find_element_by_tag_name('html') i=1 while(i<5): html.send_keys(Keys.PAGE_DOWN) i=i+1 try: WebDriverWait(driver, 400).until( EC.presence_of_element_located((By.XPATH,"//div[@id='pagelet_main_column_personal']"))) bodyC=bodyB.find_element_by_xpath("//div[@id='pagelet_main_column_personal']") WebDriverWait(driver, 400).until( EC.presence_of_element_located((By.XPATH,"//div[@class='_36d' or @id='timeline-medley']"))) bodyD=bodyC.find_element_by_xpath("//div[@class='_36d' or @id='timeline-medley']")#fine WebDriverWait(driver, 400).until( EC.presence_of_element_located((By.XPATH,"//div[@class='_2w3']"))) bodyDD=bodyD.find_element_by_xpath("//div[@class='_2w3']") except(StaleElementReferenceException): driver.get(driver.current_url) WebDriverWait(driver, 400).until( EC.presence_of_element_located((By.XPATH,"//div[@id='pagelet_main_column_personal']"))) bodyC=bodyB.find_element_by_xpath("//div[@id='pagelet_main_column_personal']") WebDriverWait(driver, 400).until( EC.presence_of_element_located((By.XPATH,"//div[@class='_36d' or @id='timeline-medley']"))) bodyD=bodyC.find_element_by_xpath("//div[@class='_36d' or @id='timeline-medley']")#fine WebDriverWait(driver, 400).until( EC.presence_of_element_located((By.XPATH,"//div[@class='_2w3']"))) bodyDD=bodyD.find_element_by_xpath("//div[@class='_2w3']") finally: pass #------photos------------------------------------------------------------------------------------------------------------- html.send_keys(Keys.PAGE_DOWN) try: WebDriverWait(driver, 400).until( EC.presence_of_element_located((By.XPATH,"//div[@id='pagelet_timeline_medley_photos']"))) element=bodyDD.find_element_by_xpath("//div[@id='pagelet_timeline_medley_photos']") facebook_dict["photos"] = element action.move_to_element(element).perform() except (NoSuchElementException, WebDriverException, StaleElementReferenceException,TimeoutException): print('photos none') element = 'None' finally: pass #------videos------------------------------------------------------------------------------------------------------------- facebook_dict["videos"] = "None" # Y a pas de block consernant les video #-----sports------------------------------------------------------------------------------------------------------------- html.send_keys(Keys.PAGE_DOWN) try: WebDriverWait(driver, 50).until( EC.presence_of_element_located((By.XPATH,"//div[@id='pagelet_timeline_medley_sports']"))) element=bodyDD.find_element_by_xpath("//div[@id='pagelet_timeline_medley_sports']") facebook_dict["sports"] = element action.move_to_element(element).perform() except (NoSuchElementException, WebDriverException, StaleElementReferenceException,TimeoutException): print('sports none') element = 'None' finally: pass #-----music------------------------------------------------------------------------------------------------------------- i=1 while(i<5): html.send_keys(Keys.PAGE_DOWN) i=i+1 try: WebDriverWait(driver, 150).until( EC.presence_of_element_located((By.XPATH,"//div[@id='pagelet_timeline_medley_music']"))) element=bodyDD.find_element_by_xpath("//div[@id='pagelet_timeline_medley_music']") facebook_dict["music"] = element action.move_to_element(element).perform() except (NoSuchElementException, WebDriverException, StaleElementReferenceException,TimeoutException): print('music none') element = 'None' finally: pass #-----book------------------------------------------------------------------------------------------------------------- i=1 while(i<5): html.send_keys(Keys.PAGE_DOWN) i=i+1 try: WebDriverWait(driver, 150).until( EC.presence_of_element_located((By.XPATH,"//div[@id='pagelet_timeline_medley_books']"))) element=bodyDD.find_element_by_xpath("//div[@id='pagelet_timeline_medley_books']") facebook_dict["books"] = element action.move_to_element(element).perform() except (NoSuchElementException, WebDriverException, StaleElementReferenceException,TimeoutException): print('books none') element = 'None' finally: pass #-------ce block pour telecharger la page i=1 while(i<5): html.send_keys(Keys.PAGE_DOWN) i=i+1 #-----likes------------------------------------------------------------------------------------------------------------- try: WebDriverWait(driver, 200).until( EC.presence_of_element_located((By.XPATH,"//div[@id='pagelet_timeline_medley_likes']"))) element=bodyDD.find_element_by_xpath("//div[@id='pagelet_timeline_medley_likes']") facebook_dict["likes"] = element action.move_to_element(element).perform() except (NoSuchElementException, WebDriverException, StaleElementReferenceException,TimeoutException): print('likes none') element = 'None' finally: pass html.send_keys(Keys.PAGE_DOWN) if(element !='None'): element=element.find_element_by_xpath("//div[@class='_5h60 _30f']") WebDriverWait(driver, 400).until( EC.presence_of_element_located((By.XPATH,"//ul[@class='uiList _153e _5k35 _620 _509- _4ki']"))) list=element.find_element_by_xpath("//ul[@class='uiList _153e _5k35 _620 _509- _4ki']") list=list.find_elements_by_tag_name('li') print(len(list)) for i in range(len(list)): case=list[i].find_elements_by_tag_name('div') print(case[6].text) print(case[7].text) print('------------') file_infos = open("scrapping.csv", "a", encoding='utf-8') try: fieldnames = ['page_name', 'page_genre'] writer = csv.DictWriter(file_infos , fieldnames=fieldnames) writer.writeheader() for i in range(len(list)): case=list[i].find_elements_by_tag_name('div') writer.writerow({'page_name' :case[6].text, 'page_genre' :case[7].text}) writer.writerow({'page_name' :'-----------------------------------------', 'page_genre' :'-----------------------------------------'}) finally: file_infos.close()