def wait_page_completion(self, driver): """ Abstract function, used to customize how the specific spider have to wait for page completion. Blank by default :param driver: :return: """ # waiting links to other users are shown so the crawl can continue get_by_xpath_or_none(driver, '//*/span/span/span[1]', wait_timeout=3)
def extracts_linkedin_users(driver, api_client): """ Gets from a page containing a list of users, all the users. For instance: https://www.linkedin.com/search/results/people/?facetCurrentCompany=[%22221027%22] :param driver: The webdriver, logged in, and located in the page which lists users. :return: Iterator on LinkedinUser. """ for i in range(1, 11): print(f'loading {i}th user') last_result_xpath = f'//li[{i}]/*/div[@class="entity-result__item"]' result = get_by_xpath_or_none(driver, last_result_xpath) if result is not None: link_elem = get_by_xpath_or_none(result, './/*[@class="app-aware-link"]') link = link_elem.get_attribute( 'href') if link_elem is not None else None print(link) name_elem = get_by_xpath_or_none(result, './/*[@class="visually-hidden"]') name = name_elem.text if name_elem is not None else None print(name) title_elem = get_by_xpath_or_none( result, './/*[@class="entity-result__primary-subtitle t-14 t-black t-normal"]' ) title = title_elem.text if name_elem is not None else None print(title) # extract_profile_id_from_url profile_id = link.split('?')[-2] profile_id = profile_id.split('/')[-1] print(profile_id) user = extract_contact_info(api_client, profile_id) print(user) yield user if link_elem is not None: driver.execute_script("arguments[0].scrollIntoView();", link_elem) elif name_elem is not None: driver.execute_script("arguments[0].scrollIntoView();", name_elem) elif title_elem is not None: driver.execute_script("arguments[0].scrollIntoView();", title_elem) else: print("Was not possible to scroll") time.sleep(random.uniform(0.7, 1.5))
def wait_page_completion(self, driver): """ Abstract function, used to customize how the specific spider have to wait for page completion. Blank by default :param driver: :return: """ # waiting links to other users are shown so the crawl can continue get_by_xpath_or_none(driver, "//*/div[@class='pv-deferred-area ember-view']", wait_timeout=3)
def wait_page_completion(self, driver): """ Abstract function, used to customize how the specific spider must wait for a search page completion. Blank by default :param driver: :return: """ profile_xpath = "//*[@id='nav-settings__dropdown-trigger']/img" get_by_xpath_or_none(driver, profile_xpath) time.sleep(random.uniform(3, 7)) pass
def parser_search_results_page(self, response): print('Now parsing search result page') no_result_found_xpath = '//*[text()="No results found."]' no_result_response = get_by_xpath_or_none(driver=self.driver, xpath=no_result_found_xpath, wait_timeout=NO_RESULT_WAIT_TIMEOUT, logs=False) if no_result_response is not None: print('"No results" message shown, stop crawling this company') return else: company = extract_company(self.driver) print(f'Company:{company}') users = extracts_linkedin_users(self.driver, company=company, api_client=self.api_client) for user in users: yield user # incrementing the index at the end of the url url = response.request.url next_url_split = url.split('=') index = int(next_url_split[-1]) next_url = '='.join(next_url_split[:-1]) + '=' + str(index + 1) yield Request(url=next_url, callback=self.parser_search_results_page, meta={'company': company}, dont_filter=True, )
def extracts_linkedin_users(driver, api_client): """ Gets from a page containing a list of users, all the users. For instance: https://www.linkedin.com/search/results/people/?facetCurrentCompany=[%22221027%22] :param driver: The webdriver, logged in, and located in the page which lists users. :return: Iterator on LinkedinUser. """ for i in range(1, 11): print(f'loading {i}th user') last_result_xpath = f'//li[{i}]/*/div[@class="search-result__wrapper"]' result = get_by_xpath_or_none(driver, last_result_xpath) if result is not None: link_elem = get_by_xpath_or_none( result, './/*[@class="search-result__result-link ember-view"]') link = link_elem.get_attribute( 'href') if link_elem is not None else None name_elem = get_by_xpath_or_none(result, './/*[@class="name actor-name"]') name = name_elem.text if name_elem is not None else None title_elem = get_by_xpath_or_none(result, './/p') title = title_elem.text if name_elem is not None else None # extract_profile_id_from_url profile_id = link.split('/')[-2] user = extract_contact_info(api_client, profile_id) yield user if link_elem is not None: driver.execute_script("arguments[0].scrollIntoView();", link_elem) elif name_elem is not None: driver.execute_script("arguments[0].scrollIntoView();", name_elem) elif title_elem is not None: driver.execute_script("arguments[0].scrollIntoView();", title_elem) else: print("Was not possible to scroll") time.sleep(0.7)
def extract_company(driver): """ Extract company name from a search result page. :param driver: The selenium webdriver. :return: The company string, None if something wrong. """ company_xpath = '//li[@class="search-s-facet search-s-facet--facetCurrentCompany inline-block ' \ 'search-s-facet--is-closed ember-view"]/form/button/div/div/h3 ' company_elem = get_by_xpath_or_none(driver, company_xpath) return company_elem.text if company_elem is not None else None
def parser_search_results_page(self, response): # getting optional callback's arguments: driver = response.meta.pop('driver') # maximum number for pagination max_page = response.meta.get('max_page', None) # stop_criteria : returns True if search must stop stop_criteria = response.meta.get('stop_criteria', None) stop_criteria_args = response.meta.get('stop_criteria_args', None) # Now parsing search result page # no_result_found_xpath = '//*[text()="No results found."]' #no_result_response = get_by_xpath_or_none(driver=driver, xpath=no_result_found_xpath, wait_timeout=NO_RESULT_WAIT_TIMEOUT) result_count_found_xpath = './/*[@class="pb2 t-black--light t-14"]' result_count_response = get_by_xpath_or_none( driver=driver, xpath=result_count_found_xpath, wait_timeout=NO_RESULT_WAIT_TIMEOUT) if result_count_response is None: # no results message shown: stop crawling this company driver.close() return else: users = extracts_linkedin_users(driver, api_client=self.api_client) for user in users: if stop_criteria is not None: if stop_criteria(user, stop_criteria_args): # if stop criteria is matched stops the crawl, and also next pages driver.close() return yield user # incrementing the index at the end of the url url = response.request.url next_url_split = url.split('=') index = int(next_url_split[-1]) next_url = '='.join(next_url_split[:-1]) + '=' + str(index + 1) if max_page is not None: if index >= max_page: driver.close() return driver.close() yield Request( url=next_url, callback=self.parser_search_results_page, meta=copy.deepcopy(response.meta), dont_filter=True, )
def process_request(self, request, spider): driver = spider.driver print('SeleniumMiddleware - getting the page') driver.get(request.url) # request.meta['driver'] = self.driver # to access driver from response print('waiting for page loading') profile_xpath = "//*[@id='nav-settings__dropdown-trigger']/img" get_by_xpath(driver, profile_xpath) # waiting links to other users are shown so the crawl can continue get_by_xpath_or_none(driver, '//*/span/span/span[1]', wait_timeout=3) print('SeleniumMiddleware - retrieving body') body = to_bytes(driver.page_source) # body must be of type bytes return HtmlResponse(driver.current_url, body=body, encoding='utf-8', request=request)
def extract_profiles_from_url(self, response): # getting optional callback's arguments: driver = response.meta.pop('driver') # maximum number for pagination max_page = response.meta.get('max_page', None) # stop_criteria : returns True if search must stop stop_criteria = response.meta.get('stop_criteria', None) stop_criteria_args = response.meta.get('stop_criteria_args', None) # Now parsing search result page # no_result_found_xpath = '//*[text()="No results found."]' #no_result_response = get_by_xpath_or_none(driver=driver, xpath=no_result_found_xpath, wait_timeout=NO_RESULT_WAIT_TIMEOUT) result_count_found_xpath = './/*[@class="pb2 t-black--light t-14"]' result_count_response = get_by_xpath_or_none( driver=driver, xpath=result_count_found_xpath, wait_timeout=NO_RESULT_WAIT_TIMEOUT) if result_count_response is None: # no results message shown: stop crawling this company driver.close() return else: for i in range(1, 11): print(f'loading {i}th user') last_result_xpath = f'//li[{i}]/*/div[@class="entity-result__item"]' result = get_by_xpath_or_none(driver, last_result_xpath) if result is not None: link_elem = get_by_xpath_or_none( result, './/*[@class="app-aware-link"]') link = link_elem.get_attribute( 'href') if link_elem is not None else None print(link) # extract_profiles_from_url profile_id = link.split('?')[-2] profile_id = profile_id.split('/')[-1] print(profile_id) image_elem = get_by_xpath_or_none( result, './/*[@class="ivm-view-attr__img--centered EntityPhoto-circle-3 lazy-image ember-view"]' ) image_link = image_elem.get_attribute( 'src') if image_elem is not None else None if image_link is not None: print(image_link) get_by_xpath(result, './/*[@class="app-aware-link"]').click() time.sleep(random.uniform(8, 14)) driver.back() #yield Request(url=link, # callback=self.extract_profiles_from_url, # dont_filter=True, # ) time.sleep(random.uniform(6, 9)) # incrementing the index at the end of the url url = response.request.url next_url_split = url.split('=') index = int(next_url_split[-1]) next_url = '='.join(next_url_split[:-1]) + '=' + str(index + 1) print(next_url) if max_page is not None: if index >= max_page: driver.close() return driver.close() yield Request( url=next_url, callback=self.extract_profiles_from_url, meta=copy.deepcopy(response.meta), dont_filter=True, )