def staff_directory(self):
HR = WebDriverWait(self.driver, 40).until(
EC.element_to_be_clickable(
(By.LINK_TEXT, Dlocator.HR_sidemenu_linktext)))
action = ActionChains(self.driver)
action.move_to_element(HR).perform()
self.driver.implicitly_wait(7)
Staff_directory = WebDriverWait(self.driver, 60).until(
EC.element_to_be_clickable(
(By.LINK_TEXT, Dlocator.Staffdirectory_sidemenu_linktext)))
Staff_directory.click()
def step_impl(context, Username, password):
WebDriverWait(context.driver, 10).until(EC.element_to_be_clickable((By.ID, 'i0116')))
Log.Search_Username_Textbox(context).clear()
Log.Search_Username_Textbox(context).send_keys(Username)
Log.Search_Nextstep_Button(context).click()
WebDriverWait(context.driver, 10).until(EC.element_to_be_clickable((By.ID, 'i0118')))
Log.Search_Password_Textbox(context).clear()
Log.Search_Password_Textbox(context).send_keys(password)
Log.Search_Signin_Button(context).click()
def checkZipcodeValidation(self, zipcode=None):
element_present = EC.element_to_be_clickable((By.ID, "zip-code-popup-opener"))
WebDriverWait(self.driver, 10).until(element_present)
self.driver.find_element_by_id("zip-code-popup-opener").click()
time.sleep(3)
self.driver.find_element_by_id("zipcode").send_keys(zipcode)
self.driver.find_element_by_id("zipcode").send_keys(Keys.ENTER)
def index_page(page):
"""
抓取索引页
:param page: 页码
"""
print('正在抓取第', page, '页')
try:
url = 'https://s.taobao.com/search?q=' + quote(KEYWORD)
print(url)
browser.get(url)
if page > 1:
input = wait.until(
EC.presence_of_element_located(
(By.CSS_SELECTOR, '#mainsrp-pager div.form > input')))
submit = wait.until(
EC.element_to_be_clickable(
(By.CSS_SELECTOR,
'#mainsrp-pager div.form > span.btn.J_Submit')))
input.clear()
input.send_keys(page)
submit.click()
wait.until(
EC.text_to_be_present_in_element(
(By.CSS_SELECTOR, '#mainsrp-pager li.item.active > span'),
str(page)))
wait.until(
EC.presence_of_element_located(
(By.CSS_SELECTOR, '.m-itemlist .items .item')))
get_products()
except TimeoutError:
index_page(page)
def should_be_successful_message(self):
page = MainPage(self.browser, link)
self.add_product_to_bag()
WebDriverWait(self.browser, 5).until(
EC.element_to_be_clickable(*ProductPageLocators.SUCCESSFUL_MESSAGE))
assert page.is_element_present(
*ProductPageLocators.SUCCESSFUL_MESSAGE),\
"There is not a message about successful editing a product"
def remove_popup_banner(self):
try:
self.driver.implicitly_wait(10)
WebDriverWait(self.driver, 20).until(
EC.element_to_be_clickable(
(By.CSS_SELECTOR, 'a.next-dialog-close'))).click()
print('==> Clicked and remove popup banner')
self.driver.implicitly_wait(5)
self.driver.execute_script("window.scrollTo(0, 200)")
sleep(5)
except Exception as e:
print('Error in clicking popup banner BTN : ' + str(e))
def testLogOut(self):
switch = False
self.username.send_keys('TestUser')
self.password.send_keys('superhemligt')
self.submit.send_keys(Keys.RETURN)
self.driver.implicitly_wait(3)
self.assertIn('Min sida', self.driver.page_source)
self.log_out = WebDriverWait(self.driver, 20).until(
EC.element_to_be_clickable((By.CLASS_NAME, 'sign-out')))
self.log_out.click()
if WebDriverWait(self.driver, 20).until(EC.url_changes):
switch = True
self.assertIs(switch, True)
def get_following_names(self, profilename, By=None):
url = "https://www.hedieh.com/"+profilename+"/"
self.driver.get(url)
time.sleep(3)
path = "/html/"
wait = WebDriverWait(self.driver, 10)
following_button = wait.until(EC.element_to_be_clickable((By.XPATH, path)))
following_button.click()
names_following = self.get_names(self.driver, profilename)
f = open("following_names.txt", "w")
for i in names_following:
f.write(i)
f.write("\n")
return names_following
def get_component(self, metodo, valor):
global EC
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
wait = WebDriverWait(self.driver, 5)
try:
element = wait.until(EC.element_to_be_clickable((metodo, valor)))
return element
except:
time.sleep(2)
try:
self.component = self.driver.find_element(metodo, valor)
return self.component
except:
#print "Busca por elemento excedeu o tempo maximo.",valor
return None
def unfollow_with_file(self, filename, By=None):
f = open(filename, "r")
lines = f.readlines()
not_following_back = []
for i in lines:
not_following_back.append(i[:-1])
wait = WebDriverWait(self.driver, 10)
for i in not_following_back:
url = "https://www.hedieh.com/"+i+"/"
self.driver.get(url)
unfollow1 = wait.until(
EC.presence_of_all_elements_located((By.CSS_SELECTOR, 'button')))
unfollow1[1].click()
time.sleep(4)
path = "/html/"
unfollow2 = wait.until(
EC.element_to_be_clickable((By.XPATH, path)))
unfollow2.click()
time.sleep(randint(5,13))
def waitForElement(self,
locator,
locatorType="id",
timeout=10,
pollFrequency=0.5):
element = None
try:
byType = self.getByType(locatorType)
self.log.info("Waiting for maximum :: " + str(timeout) +
" :: seconds for element to be clickable")
wait = WebDriverWait(self.driver,
timeout=timeout,
poll_frequency=pollFrequency,
ignored_exceptions=[
NoSuchElementException,
ElementNotVisibleException,
ElementNotSelectableException
])
element = wait.until(EC.element_to_be_clickable((byType, locator)))
self.log.info("Element appeared on the web page")
except:
self.log.info("Element not appeared on the web page")
print_stack()
return element
def waitHomeButton(self):
element_present = EC.element_to_be_clickable((By.LINK_TEXT, "Home"))
WebDriverWait(self.driver, 100).until(element_present)
def clickSendButton(self):
wait = WebDriverWait(self.driver, 50)
sendButton = wait.until(
EC.element_to_be_clickable(
(By.CSS_SELECTOR, "[data-test-id='resendBtn']")))
sendButton.click()
def task_1_add_product(self, driver):
self.wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, OKULARY)))
self.driver.find_element_by_css_selector(OKULARY).clear()
self.driver.find_element_by_css_selector(OKULARY).send_keys(3)
self.driver.find_element_by_css_selector(ADD_BUTTON)
return self.driver
def clickChangeLocation(self):
element_present = EC.element_to_be_clickable((By.ID, "zip-code-popup-opener"))
WebDriverWait(self.driver, 150).until(element_present)
self.driver.find_element_by_id("zip-code-popup-opener").click()
def logOut(self):
element_present = EC.element_to_be_clickable(
(By.ID, "ctl00_uxLoginStatus"))
WebDriverWait(self.driver, 20).until(element_present)
self.driver.find_element_by_link_text("LOGOUT").click()
def clickOnElement(self, locator=None):
wait = WebDriverWait(self.driver, 100)
element = wait.until(
EC.element_to_be_clickable((By.CSS_SELECTOR, locator)))
element.click()
def get_element(
self, xpath
): #this method is the provider of waits still load any elements
return self.__driver_wait.until(
EC.element_to_be_clickable(By.XPATH, xpath))
def click(self, locator):
wait = WebDriverWait(self.driver, 15)
clickCategory = wait.until(
EC.element_to_be_clickable((By.CSS_SELECTOR, locator)))
clickCategory.click()
def scroll_down(self):
global position
try:
wait = WebDriverWait(self.driver, 10)
wait.until(
EC.element_to_be_clickable(
(By.CSS_SELECTOR,
'#form-cheapest-acc-hot13445 > button:nth-child(1)')))
except TimeoutException:
print("Loading took too much time!")
try:
print('Scrolling page ...')
# self.driver.execute_script('window.scrollTo(0, document.body.scrollHeight);')
read_mores = self.driver.find_elements_by_xpath('//form/button')
for read_more in read_mores:
self.driver.execute_script("arguments[0].scrollIntoView();",
read_more)
# read_more.click()
sleep(1)
# even 25 seconds may not be enough to load the page!
# todo AJAX: maximum 30 seconds
print('Waiting ajax full load ...')
sleep(11) # wait ajax full load
print("Scraping page ...")
soup = BeautifulSoup(self.driver.page_source, 'lxml')
hotel_list = soup.find_all(
'article', {'class': 'crosselling-line availability-item'})
euro_symbol = '€'
print("\n\tSnapshoot:\n")
try:
for i, hotel in enumerate(hotel_list):
self.all_positions.append(i + 1)
hotel_name = hotel.find('a', {
'data-tl': 'acc-title'
}).getText()
# fixme: remove whitespaces REF: https://stackoverrun.com/es/q/743639
hotel_name = ' '.join(hotel_name.split())
self.all_hotels.append(hotel_name)
# print(" %d - %s" % (i + 1, hotel_name))
hotel_zone = hotel.find('span', {
'class': '_hotelzone'
}).getText().strip(',')
hotel_name = ' '.join(hotel_name.split())
self.all_zones.append(hotel_zone)
hotel_integer = hotel.find('span', {
'class': 'hotel-price'
}).getText()
hotel_decimal = hotel.find('span', {
'class': 'hotel-price-decimal'
}).getText().strip('€')
hotel_integer = hotel_integer.replace(',', '')
hotel_price = '{}.{}'.format(hotel_integer, hotel_decimal)
hotel_price = float(hotel_price)
hotel_price = "{0:.2f}".format(hotel_price)
self.all_prices.append(hotel_price)
if len(hotel_price) == 5:
hotel_price = " " + hotel_price
if len(hotel_price) == 6:
hotel_price = " " + hotel_price
if len(hotel_price) == 7:
hotel_price = " " + hotel_price
if len(hotel_price) == 8:
hotel_price = "" + hotel_price
if i < 9:
print(" %d - %s %s %s - %s" %
(i + 1, hotel_price, euro_symbol, hotel_name,
hotel_zone))
else:
print("%d - %s %s %s - %s" %
(i + 1, hotel_price, euro_symbol, hotel_name,
hotel_zone))
print("\n\tRanking:\n")
# float cast
new_prices = []
for element in self.all_prices:
rank = float(element)
new_prices.append(rank)
# final list
display_list = list(
zip(self.all_positions, self.all_hotels, new_prices,
self.all_zones))
ranking = sorted(display_list, key=operator.itemgetter(2))
# todo REF: https://discuss.codecademy.com/t/how-can-i-sort-a-zipped-object/454412/6
for j, k, v, w in ranking:
if v < 100.00:
print(" ", "{0:.2f}".format(v), k)
if 99.00 < v < 1000.00:
print(" ", "{0:.2f}".format(v), k)
if 999.00 < v < 10000.00:
print(" ", "{0:.2f}".format(v), k)
if v > 9999.00:
print("", "{0:.2f}".format(v), k)
self.cheap = ranking[0]
self.data = ranking
print('\nCheapest reservation: ', self.cheap[1], self.cheap[2],
euro_symbol)
# self.display = display_list[7]
# print('Target button number: ', self.display.index(self.cheap[0]))
self.display = display_list
for i, collation in enumerate(display_list):
if collation[1] == self.cheap[1]:
position = i
print('Pointing to the target button ', position + 1, ' ...')
self.index = str(position)
if self.error is False:
self.target_button(self.index)
sleep(2)
except Exception as e:
self.error = True
print(e)
print(
'Some error occurred while trying to scratch the hotel list'
)
# activate page analyzer
if mode == "d":
print('Downloading html to computer ...')
file = open("bedsonline.html", "w")
file.write(soup.prettify())
file.close()
except NoSuchElementException:
print('Some error occurred while trying to scroll down')
self.error = True
def search_target_profile(self):
try:
# todo: filling the search bar
# search_bar.send_keys(self.target_username).click # this line is wrong, read below please
# fixme: you cannot enter text directly use the autocomplete square icon to the right of the field
WebDriverWait(self.driver, 100).until(
EC.visibility_of_element_located(
(By.XPATH,
'//section/div/form/fieldset[1]/div/a/span'))).click()
# todo: https://dzone.com/articles/perform-actions-using-javascript-in-python-seleniu
# Method 1: Executing JavaScript at Document Root Level
# javaScript = "document.getElementsByClassName('ui-dialog-titlebar-close ui-corner-all')[0].click();"
# self.driver.execute_script(javaScript)
# Method 2: Executing JavaScript at Element Level:
print("Manipulating search engine ...")
try:
wait = WebDriverWait(self.driver, 100)
wait.until(
EC.element_to_be_clickable(
(By.CSS_SELECTOR,
"#continent-picker-tab > li:nth-child"
"(" + self.target_continent + ") > a:nth-child(1)")))
except TimeoutException:
print("Searching took too much time!")
if self.target_continent != '4':
picker = WebDriverWait(self.driver, 100).until(
EC.visibility_of_element_located(
(By.CSS_SELECTOR,
"#continent-picker-tab > li:nth-child(" +
self.target_continent + ") > a:nth-child(1)")))
self.driver.execute_script("arguments[0].click();", picker)
sleep(1)
picker = WebDriverWait(self.driver, 100).until(
EC.visibility_of_element_located(
(By.XPATH,
"//section/article[1]/div/ul[" + self.target_country_col +
"]/li[" + self.target_country_row + "]/a")))
self.driver.execute_script("arguments[0].click();", picker)
sleep(1)
picker = WebDriverWait(self.driver, 100).until(
EC.visibility_of_element_located(
(By.XPATH,
"//section/article[2]/div/ul[" + self.target_city_col +
"]/li[" + self.target_city_row + "]/a")))
self.driver.execute_script("arguments[0].click();", picker)
sleep(1)
picker = WebDriverWait(self.driver, 100).until(
EC.visibility_of_element_located(
(By.XPATH, "//article[3]/div/ul[1]/li[1]/a")))
self.driver.execute_script("arguments[0].click();", picker)
sleep(1)
# search button
login_button = WebDriverWait(self.driver, 100).until(
EC.visibility_of_element_located(
(By.XPATH, '//*[@id="mainsearch"]')))
print('Loading page ...')
login_button.submit()
except Exception:
self.error = True
print('Could not find search bar')
).click()
time.sleep(4)
driver.find_element_by_xpath("//*[contains(text(), 'Choice2')]").click()
time.sleep(4)
print("Select checkbox Multiple choice")
driver.find_element(By.ID, "checkbox-00000018-3").click()
time.sleep(2)
driver.find_element(
By.XPATH,
"//body/form[@id='form']/div/div/div[9]/div[1]/div[4]/div[1]/input[1]"
).send_keys("testing 1")
# print("Verification")
driver.execute_script(
"arguments[0].click();",
WebDriverWait(driver, 10).until(
EC.element_to_be_clickable((
By.XPATH,
"//span[@class='recaptcha-checkbox goog-inline-block recaptcha-checkbox-unchecked rc-anchor-checkbox']/div[@class='recaptcha-checkbox-checkmark']"
))))
time.sleep(4)
print("Submitting form")
driver.find_element(
By.XPATH,
"/html[1]/body[1]/form[1]/div[1]/div[3]/div[2]/button[1]").click()
time.sleep(3)
# closing browser tab
driver.close()
def waitElement1(self, locator=None):
element_present = EC.element_to_be_clickable(
(By.CSS_SELECTOR, locator))
WebDriverWait(self.driver, 100).until(element_present)
def Copy(self):
time.sleep(10)
copy = WebDriverWait(self.browser, 10).until(
EC.element_to_be_clickable(self.Locator_login_buttons["Copy"]))
copy.click()
def step_impl(context):
HomePage.Open(context)
WebDriverWait(context.driver, 10).until(EC.element_to_be_clickable((By.ID, "i0116")))
logpage.Signin_Go(context).click
def linkdin(name):
browser = webdriver.Chrome('chromedriver.exe')
wait = WebDriverWait(browser, 30)
browser.get('https://www.linkedin.com/uas/login')
username = "******"
password = "******"
search_item = name
elementID = browser.find_element_by_id('username')
elementID.send_keys(username)
elementID = browser.find_element_by_id('password')
elementID.send_keys(password)
elementID.submit()
wait.until(EC.element_to_be_clickable((By.ID, "nav-typeahead-wormhole")))
browser.find_element_by_css_selector("div.nav-search-typeahead input").send_keys(search_item)
browser.find_element_by_css_selector("div.nav-search-typeahead input").send_keys(Keys.ENTER)
profilesQueued = []
wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, "span.name.actor-name")))
wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, "h3.search-results__total"))) # Pour ne pas avoir aucun resultats
scheight = .1
while scheight < 1.0:
browser.execute_script("window.scrollTo(0, document.body.scrollHeight*%s);" % scheight)
scheight += .1
try:
print(browser.find_element_by_css_selector('div.search-result__wrapper').text)
except(NoSuchElementException):
print('No div.div.search-result__wrapper')
finally:
pass
profils = browser.find_elements_by_css_selector("div.search-result__wrapper")
name_location = profils[0].find_element_by_css_selector("span.name.actor-name")
browser.execute_script("arguments[0].scrollIntoView({behavior: 'smooth', block: 'center', inline: 'nearest'});", name_location)
name = name_location.text.encode('utf-8')
#name = name.lower()
name = name.decode()
if search_item in name:
url_profil = profils[0].find_element_by_css_selector("a.search-result__result-link").get_attribute('href')
profilesQueued.append(url_profil)
infoo = {
"name_linkdin": "None",
"profile_title_linkdin": "None",
"loc": "None",
"information": "None"
}
educationn = {
"college_name": "None",
"degree_name": "None",
"degree_year": "None"
}
experiencee = {
"company_name": "None",
"job_title": "None",
"joining_date": "None",
"exp": "None"
}
linkdin_dict = {
"info": "None",
"education": "None",
"experience": "None",
"competences": "None"
}
if (len(profilesQueued) == 1):
fullLink = profilesQueued.pop()
browser.get(fullLink)
SCROLL_PAUSE_TIME = 5
last_height = browser.execute_script("return document.body.scrollHeight")
# Get scroll height
for i in range(3):
browser.execute_script("window.scrollTo(0, document.body.scrollHeight);")
time.sleep(SCROLL_PAUSE_TIME)
new_height = browser.execute_script("return document.body.scrollHeight")
if new_height == last_height:
break
last_height = new_height
## Info
soup = BeautifulSoup(browser.page_source, "html.parser")
name_div = soup.find('div', {'class': 'flex-1 mr5'})
if name_div is not None:
name_loc = name_div.find_all('ul')
if name_loc is not None:
name = name_loc[0].find('li').get_text().strip()
loc = name_loc[1].find('li').get_text().strip()
infoo["name_linkdin"] = name
infoo["loc"] = loc
else:
name_loc = 'None'
profile_title = name_div.find('h2').get_text().strip()
infoo["profile_title_linkdin"] = profile_title
else:
name_div = 'None'
##Information
name_information = soup.find('div', {'class': 'pv-oc ember-view'})
if name_information is not None:
content_information = name_information.find('p')
if content_information is not None:
content_information = content_information.get_text().strip()
content_information = content_information.replace('\n','')
content_information = content_information.replace('voir plus','')
infoo["information"] = content_information
else:
content_information = 'None'
infoo["information"] = content_information
linkdin_dict["info"] = infoo
## Experience
# testExp = exp_section.get_text().strip()
# info.append(testExp)
try:
exp_section = soup.find('section', {'id': 'experience-section'})
try:
exp_section = exp_section.find('ul')
try:
li_tags = exp_section.find('div')
try:
a_tags = li_tags.find('a')
try:
job_title = a_tags.find('h3').get_text().strip()
except (NoSuchElementException, WebDriverException, StaleElementReferenceException):
job_title = 'None'
finally:
pass
experiencee["job_title"] = job_title
try:
company_name = a_tags.find_all('p')[1].get_text().strip()
except (NoSuchElementException, WebDriverException, StaleElementReferenceException):
company_name = 'None'
finally:
pass
experiencee["company_name"] = company_name
try:
joining_date = a_tags.find_all('h4')[0].find_all('span')[1].get_text().strip()
except (NoSuchElementException, WebDriverException, StaleElementReferenceException):
joining_date ='None'
finally:
pass
experiencee["joining_date"] = joining_date
try:
exp = a_tags.find_all('h4')[1].find_all('span')[1].get_text().strip()
except (NoSuchElementException, WebDriverException, StaleElementReferenceException):
exp = 'None'
finally:
pass
experiencee["exp"] = exp
except (NoSuchElementException, WebDriverException, StaleElementReferenceException):
a_tags = 'None'
finally:
pass
except (NoSuchElementException, WebDriverException, StaleElementReferenceException):
li_tags = 'None'
finally:
pass
except (AttributeError, NoSuchElementException, WebDriverException, StaleElementReferenceException):
exp_section = 'None'
finally:
pass
except (AttributeError, NoSuchElementException, WebDriverException, StaleElementReferenceException):
exp_section = 'None'
finally:
pass
linkdin_dict["experience"] = experiencee
## Formation
browser.page_source
scheight = .1
while scheight < 1.0:
browser.execute_script("window.scrollTo(0, document.body.scrollHeight*%s);" % scheight)
scheight += .1
try:
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, 'section#education-section')))
try:
edu_section = browser.find_element_by_css_selector("section#education-section")
# testExp = edu_section.text
# info.append(testExp)
try:
college_name = edu_section.find_element_by_css_selector('h3.pv-entity__school-name.t-16.t-black.t-bold').text
except (NoSuchElementException, WebDriverException, StaleElementReferenceException):
college_name = 'None'
finally:
pass
educationn["college_name"] = college_name
try:
degree_name = edu_section.find_element_by_css_selector('p.pv-entity__secondary-title.pv-entity__degree-name.t-14.t-black.t-normal').text
except (NoSuchElementException, WebDriverException, StaleElementReferenceException):
degree_name = 'None'
finally:
pass
educationn["degree_name"] = degree_name
try:
degree_year = edu_section.find_element_by_css_selector('p.pv-entity__dates.t-14.t-black--light.t-normal')
degree_year = degree_year.find_element_by_css_selector('span:nth-child(2)').text
except (NoSuchElementException, WebDriverException, StaleElementReferenceException):
degree_year = 'None'
finally:
pass
educationn["degree_year"] = degree_year
except (NoSuchElementException, WebDriverException, StaleElementReferenceException):
edu_section = 'None'
finally:
pass
except:
print('No education')
finally:
pass
linkdin_dict["education"] = educationn
# Competences Professionnelles
browser.page_source
scheight = .1
while scheight < 1.0:
browser.execute_script("window.scrollTo(0, document.body.scrollHeight*%s);" % scheight)
scheight += .1
try:
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, 'section.pv-profile-section.pv-skill-categories-section.artdeco-container-card.ember-view')))
try:
competence_section = browser.find_element_by_css_selector('section.pv-profile-section.pv-skill-categories-section.artdeco-container-card.ember-view')
button_skills = competence_section.find_element_by_css_selector('button.pv-profile-section__card-action-bar.pv-skills-section__additional-skills.artdeco-container-card-action-bar.artdeco-button.artdeco-button--tertiary.artdeco-button--3.artdeco-button--fluid')
button_skills.send_keys(Keys.ENTER)
competence_section = browser.find_element_by_css_selector('section.pv-profile-section.pv-skill-categories-section.artdeco-container-card.ember-view')
try:
competence_section_1 = competence_section.find_element_by_css_selector('ol.pv-skill-categories-section__top-skills.pv-profile-section__section-info.section-info.pb1')
competence_section_1 = competence_section_1.text.split(",")
except:
competence_section_1 = 'None'
finally:
pass
try:
competence_section_2 = competence_section.find_element_by_css_selector('div#skill-categories-expanded')
competence_div = competence_section_2.find_element_by_css_selector('div.pv-skill-category-list.pv-profile-section__section-info.mb6.ember-view')
competence_ol = competence_div.find_element_by_css_selector('ol.pv-skill-category-list__skills_list.list-style-none')
competence_ol = competence_ol.text.split(",")
competence_section_1 = competence_ol + competence_section_1
except:
competence_section_2 = 'None'
finally:
pass
except:
competence_section = 'None'
competence_section_1 = 'None'
finally:
pass
except:
print('No skills')
competence_section_1 = 'None'
finally:
pass
linkdin_dict["competences"] = competence_section_1
else:
message='No profil fineded'
print(linkdin_dict)
browser.close()
return linkdin_dict
def task_1(self, driver):
self.wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, ZAD_1)))
self.driver.find_element_by_css_selector(ZAD_1).click()
return self.driver
def waitChangeLocation(self):
element_present = EC.element_to_be_clickable((By.ID, "zip-code-popup-opener"))
WebDriverWait(self.driver, 150).until(element_present)
def esperar_elemento(self, locator, MyTextElement=None):
Get_Entity = Functions.get_entity(self, locator)
if Get_Entity is None:
return print("No se encontro el Valor en el Json definido")
else:
try:
if self.json_GetFieldBy.lower() == "id":
wait = WebDriverWait(self.driver, 20) #TIEMPO DE ESPERA
wait.until(
EC.visibility_of_element_located(
(By.ID, self.json_ValueToFind)))
wait.until(
EC.element_to_be_clickable(
(By.ID, self.json_ValueToFind)))
print(u"Esperar_Elemento: Se visualizo el elemento " +
locator)
return True
if self.json_GetFieldBy.lower() == "name":
wait = WebDriverWait(self.driver, 20)
wait.until(
EC.isibility_of_element_located(
(By.NAME, self.json_ValueToFind)))
wait.until(
EC.isibility_of_element_located(
(By.NAME, self.json_ValueToFind)))
print(u"Esperar_Elemento: Se visualizo el elemento " +
locator)
return True
if self.json_GetFieldBy.lower() == "xpath":
wait = WebDriverWait(self.driver, 20)
if MyTextElement is not None:
self.json_ValueToFind = self.json_ValueToFind.format(
MyTextElement)
print(self.json_ValueToFind)
wait.until(
EC.visibility_of_element_located(
(By.XPATH, self.json_ValueToFind)))
wait.until(
EC.element_to_be_clickable(
(By.XPATH, self.json_ValueToFind)))
print(u"Esperar_Elemento: Se visualizo el elemento " +
locator)
return True
if self.json_GetFieldBy.lower() == "link":
wait = WebDriverWait(self.driver, 20)
wait.until(
EC.visibility_of_element_located(
(By.LINK_TEXT, self.json_ValueToFind)))
wait.until(
EC.EC.element_to_be_clickable(
(By.PARTIAL_LINK_TEXT, self.json_ValueToFind)))
print(u"Esperar_Elemento: Se visualizo el elemento " +
locator)
return True
except TimeoutError:
print(u"Esperar_Elemento: No presente " + locator)
Functions.tearDow(self)
except NoSuchElementException(self):
print(u"Esperar_Elemento: No presente " + locator)
def facebook(name):
username='******'
password='******'
url='https://www.facebook.com/'
try:
chrome_options = webdriver.ChromeOptions()
prefs = {"profile.default_content_setting_values.notifications" : 2}
chrome_options.add_experimental_option("prefs",prefs)
driver = webdriver.Chrome(chrome_options=chrome_options)
#action = webdriver.ActionChains(driver)
#pour vider les fichiers---------------------------------------------------------------------------
mon_fichier = open("fichier.txt", "w", encoding="utf-8")
mon_fichier.write("")
mon_fichier.close()
facebook_dict = {
"photos": "None",
"videos": "None",
"sports": "None",
"music": "None",
"books": "None",
"likes": "None"
}
try:
driver.get(url)
#etape d'authentification---------------------------------------------------------------------------
#----------------------------------------------------------------------------------------------------
driver.find_element_by_id('email').send_keys(username)
driver.find_element_by_id('pass').send_keys(password)
driver.find_element_by_id('loginbutton').click()
except(NoSuchElementException, WebDriverException, StaleElementReferenceException):
driver.close()
chrome_options = webdriver.ChromeOptions()
prefs = {"profile.default_content_setting_values.notifications" : 2}
chrome_options.add_experimental_option("prefs",prefs)
driver = webdriver.Chrome(chrome_options=chrome_options)
driver.get(url)
#etape d'authentification---------------------------------------------------------------------------
#----------------------------------------------------------------------------------------------------
driver.find_element_by_id('email').send_keys(username)
driver.find_element_by_id('pass').send_keys(password)
driver.find_element_by_id('loginbutton').click()
finally:
pass
#etape de recherche---------------------------------------------------------------------------
#----------------------------------------------------------------------------------------------------
wait = WebDriverWait(driver, 30)
wait.until( EC.presence_of_element_located((By.XPATH,"//input[@data-testid='search_input' or @name='q']")))
element=driver.find_element_by_xpath("//input[@data-testid='search_input' or @name='q']")
element.send_keys('bassma maachi')
driver.find_element_by_xpath("//button[@data-testid='facebar_search_button']").click()
wait.until( EC.element_to_be_clickable((By.XPATH,"//a[@data-testid='see_more_header-ENTITY_USER-user']")))
driver.find_element_by_xpath("//a[@data-testid='see_more_header-ENTITY_USER-user']").click()
#etape de cibler les resultats---------------------------------------------------------------------------
#----------------------------------------------------------------------------------------------------
#wait.until( EC.presence_of_element_located((By.XPATH,"//div[@class='clearfix pbm']")))
element_from=driver.find_element_by_xpath("//div[@class='clearfix pbm']")
wait.until( EC.presence_of_element_located((By.XPATH,"//div[@class='_1qkq _1qkx']")))
element_from1=element_from.find_element_by_xpath("//div[@class='_1qkq _1qkx']")
wait.until( EC.presence_of_element_located((By.XPATH,"//div[@class='_1yt' or @id='BrowseResultsContainer']")))
element_from2=element_from1.find_element_by_xpath("//div[@class='_1yt' or @id='BrowseResultsContainer']")
wait.until( EC.presence_of_element_located((By.XPATH,"//div[@class='_4p2o _87m1']")))
try:
wait.until( EC.presence_of_element_located((By.XPATH,"//div[@class='_4p2o _87m1']")))
wait.until( EC.presence_of_element_located((By.XPATH,"//div[@class='_4p2o _87m1']")))
list=element_from2.find_elements_by_xpath("//div[@class='_4p2o _87m1']")
except(StaleElementReferenceException):
driver.get(driver.current_url)
list=element_from2.find_elements_by_xpath("//div[@class='_4p2o _87m1']")
finally:
pass
#etape de stockage---------------------------------------------------------------------------
#----------------------------------------------------------------------------------------------------
mon_fichier = open("fichier.txt", "a", encoding="utf-8")
for i in range(len(list)):
element_from3=list[i]
WebDriverWait(driver, 150).until( EC.presence_of_element_located((By.TAG_NAME,"span")))
spane=(element_from3.find_elements_by_tag_name('span'))[0]
s_text=spane.text
mon_fichier.write(s_text+"\n")
WebDriverWait(driver, 150).until( EC.presence_of_element_located((By.TAG_NAME,"img")))
image=((element_from3.find_elements_by_tag_name('img'))[0]).get_attribute("src")
mon_fichier.write(image+"\n")
mon_fichier.close()
#etape :choix de profil---------------------------------------------------------------------------
#----------------------------------------------------------------------------------------------------
profile=list[0]
profile_elem=profile.find_element_by_xpath("//a[@class='_32mo']")
profile_elem.click()
print(driver.current_url)
WebDriverWait(driver, 400).until( EC.presence_of_element_located((By.XPATH,"//div[@class='clearfix _ikh _3-8y']")))
element=driver.find_element_by_xpath("//div[@class='clearfix _ikh _3-8y']")
element1=element.find_element_by_xpath("//div[@class='_3-96']")
profile_barre=element1.find_element_by_xpath("//ul[@class='uiList _3-8x _2pic _4kg' or @id='u_0_1n']")
barre_infos=profile_barre.find_elements_by_tag_name('li')
print(len(barre_infos))
#etape :stockage dans le fichier---------------------------------------------------------------------------
#----------------------------------------------------------------------------------------------------
file_infos = open("scrapping.csv", "w", encoding='utf-8')
try:
writer = csv.writer(file_infos)
writer.writerow( ['information-----------------------'] )
for i in range(len(barre_infos)):
writer.writerow( [barre_infos[i].text] )
print(barre_infos[i].text)
writer.writerow( ['----------------------------------'] )
finally:
file_infos.close()
#etape :about page---------------------------------------------------------------------------
#----------------------------------------------------------------------------------------------------
profile_barre=driver.find_element_by_xpath("//div[@class='clearfix' or @id='fbTimelineHeadline']")
profile_ul=profile_barre.find_element_by_xpath("//ul[@class='_6_7 clearfix' or @id='u_0_12']")
list=profile_ul.find_elements_by_tag_name('li')
barre_infos=list[1].find_element_by_tag_name('a')
barre_infos.click()
print(driver.current_url)
#on est dans la page about-----------------------------on va cibler la partir 'mention de j'aime'-----------------
WebDriverWait(driver, 400).until( EC.presence_of_element_located((By.XPATH,"//div[@class='fb_content clearfix' or @id='content']")))
body=driver.find_element_by_xpath("//div[@class='fb_content clearfix' or @id='content']")
bodyA=body.find_element_by_xpath("//div[@id='mainContainer']")
bodyB=bodyA.find_element_by_xpath("//div[@class='clearfix' or @id='contentCol']")
html = driver.find_element_by_tag_name('html')
i=1
while(i<5):
html.send_keys(Keys.PAGE_DOWN)
i=i+1
try:
WebDriverWait(driver, 400).until( EC.presence_of_element_located((By.XPATH,"//div[@id='pagelet_main_column_personal']")))
bodyC=bodyB.find_element_by_xpath("//div[@id='pagelet_main_column_personal']")
WebDriverWait(driver, 400).until( EC.presence_of_element_located((By.XPATH,"//div[@class='_36d' or @id='timeline-medley']")))
bodyD=bodyC.find_element_by_xpath("//div[@class='_36d' or @id='timeline-medley']")#fine
WebDriverWait(driver, 400).until( EC.presence_of_element_located((By.XPATH,"//div[@class='_2w3']")))
bodyDD=bodyD.find_element_by_xpath("//div[@class='_2w3']")
except(StaleElementReferenceException):
driver.get(driver.current_url)
WebDriverWait(driver, 400).until( EC.presence_of_element_located((By.XPATH,"//div[@id='pagelet_main_column_personal']")))
bodyC=bodyB.find_element_by_xpath("//div[@id='pagelet_main_column_personal']")
WebDriverWait(driver, 400).until( EC.presence_of_element_located((By.XPATH,"//div[@class='_36d' or @id='timeline-medley']")))
bodyD=bodyC.find_element_by_xpath("//div[@class='_36d' or @id='timeline-medley']")#fine
WebDriverWait(driver, 400).until( EC.presence_of_element_located((By.XPATH,"//div[@class='_2w3']")))
bodyDD=bodyD.find_element_by_xpath("//div[@class='_2w3']")
finally:
pass
#------photos-------------------------------------------------------------------------------------------------------------
html.send_keys(Keys.PAGE_DOWN)
try:
WebDriverWait(driver, 400).until( EC.presence_of_element_located((By.XPATH,"//div[@id='pagelet_timeline_medley_photos']")))
element=bodyDD.find_element_by_xpath("//div[@id='pagelet_timeline_medley_photos']")
facebook_dict["photos"] = element
action.move_to_element(element).perform()
except (NoSuchElementException, WebDriverException, StaleElementReferenceException,TimeoutException):
print('photos none')
element = 'None'
finally:
pass
#------videos-------------------------------------------------------------------------------------------------------------
facebook_dict["videos"] = "None" # Y a pas de block consernant les video
#-----sports-------------------------------------------------------------------------------------------------------------
html.send_keys(Keys.PAGE_DOWN)
try:
WebDriverWait(driver, 50).until( EC.presence_of_element_located((By.XPATH,"//div[@id='pagelet_timeline_medley_sports']")))
element=bodyDD.find_element_by_xpath("//div[@id='pagelet_timeline_medley_sports']")
facebook_dict["sports"] = element
action.move_to_element(element).perform()
except (NoSuchElementException, WebDriverException, StaleElementReferenceException,TimeoutException):
print('sports none')
element = 'None'
finally:
pass
#-----music-------------------------------------------------------------------------------------------------------------
i=1
while(i<5):
html.send_keys(Keys.PAGE_DOWN)
i=i+1
try:
WebDriverWait(driver, 150).until( EC.presence_of_element_located((By.XPATH,"//div[@id='pagelet_timeline_medley_music']")))
element=bodyDD.find_element_by_xpath("//div[@id='pagelet_timeline_medley_music']")
facebook_dict["music"] = element
action.move_to_element(element).perform()
except (NoSuchElementException, WebDriverException, StaleElementReferenceException,TimeoutException):
print('music none')
element = 'None'
finally:
pass
#-----book-------------------------------------------------------------------------------------------------------------
i=1
while(i<5):
html.send_keys(Keys.PAGE_DOWN)
i=i+1
try:
WebDriverWait(driver, 150).until( EC.presence_of_element_located((By.XPATH,"//div[@id='pagelet_timeline_medley_books']")))
element=bodyDD.find_element_by_xpath("//div[@id='pagelet_timeline_medley_books']")
facebook_dict["books"] = element
action.move_to_element(element).perform()
except (NoSuchElementException, WebDriverException, StaleElementReferenceException,TimeoutException):
print('books none')
element = 'None'
finally:
pass
#-------ce block pour telecharger la page
i=1
while(i<5):
html.send_keys(Keys.PAGE_DOWN)
i=i+1
#-----likes-------------------------------------------------------------------------------------------------------------
try:
WebDriverWait(driver, 200).until( EC.presence_of_element_located((By.XPATH,"//div[@id='pagelet_timeline_medley_likes']")))
element=bodyDD.find_element_by_xpath("//div[@id='pagelet_timeline_medley_likes']")
facebook_dict["likes"] = element
action.move_to_element(element).perform()
except (NoSuchElementException, WebDriverException, StaleElementReferenceException,TimeoutException):
print('likes none')
element = 'None'
finally:
pass
html.send_keys(Keys.PAGE_DOWN)
if(element !='None'):
element=element.find_element_by_xpath("//div[@class='_5h60 _30f']")
WebDriverWait(driver, 400).until( EC.presence_of_element_located((By.XPATH,"//ul[@class='uiList _153e _5k35 _620 _509- _4ki']")))
list=element.find_element_by_xpath("//ul[@class='uiList _153e _5k35 _620 _509- _4ki']")
list=list.find_elements_by_tag_name('li')
print(len(list))
for i in range(len(list)):
case=list[i].find_elements_by_tag_name('div')
print(case[6].text)
print(case[7].text)
print('------------')
file_infos = open("scrapping.csv", "a", encoding='utf-8')
try:
fieldnames = ['page_name', 'page_genre']
writer = csv.DictWriter(file_infos , fieldnames=fieldnames)
writer.writeheader()
for i in range(len(list)):
case=list[i].find_elements_by_tag_name('div')
writer.writerow({'page_name' :case[6].text, 'page_genre' :case[7].text})
writer.writerow({'page_name' :'-----------------------------------------', 'page_genre' :'-----------------------------------------'})
finally:
file_infos.close()