def login_linkedinx(email, password, driver=None, pincode=None, cur=None, owner_id=None, task_id=None, task_type=None): print("==== LOGIN =====") lastrun_date = datetime.now() completed_date = datetime.now() login_linkedin_withwebdriver(email, password, driver) print('get login_linkedin_withwebdriver driver') wait = WebDriverWait(driver, IDLE_INTERVAL_IN_SECONDS) # check if user is exist bot_status = botstatus.DONE if exist_user(wait): print("That user is an existing user.") if check_password_correct(wait): print('pw is correct') sleep(1000) res = check_captcha(driver)
def sale_nav_search(email, password, url, limit=1000): driver = login_linkedin_withwebdriver(email, password) max_count = limit print('url:', url) # url = url.replace('search?keywords', 'search/results?keywords') offset = 0 mh = 100 for i in range( limit // mh ): offset = mh * i pageurl = url + "&count={count}&start={start}".format(count=mh, start=offset) #get_search_contact_salesurls(pageurl, driver, None, None, None, i, cookies) driver.get(pageurl) time.sleep(5) """ soup = BeautifulSoup(driver.page_source, "html.parser") members = soup.find_all('li', class_='result') for member in members: # name and id_ tag_a = member.find('a', class_='name-link') print( member) # info """ driver.close()
def login_linkedin(email, password, pincode=None, cur=None, owner_id=None, task_id=None, task_type=None): print("==== LOGIN =====") lastrun_date = datetime.now() completed_date = datetime.now() driver = login_linkedin_withwebdriver(email, password) try: wait = WebDriverWait(driver, IDLE_INTERVAL_IN_SECONDS) # check if user is exist bot_status = botstatus.DONE if exist_user(wait): print("That user is an existing user.") if check_password_correct(wait): print('pw is correct') res = check_captcha(driver) if res == False: driver.close() return bot_status, lastrun_date, completed_date # pin code verification res, no_pin_required = pinverify(wait, pincode, cur, owner_id, task_id, task_type) if res and no_pin_required: completed_date = datetime.now() driver.close() return bot_status, lastrun_date, completed_date, completed_date if (no_pin_required == False): print("Pin to verify!") if pincode is None: bot_status = botstatus.PIN_REQUIRED else: bot_status = botstatus.PIN_INVALID else: print("sucessfull login without pin code verification!") else: print('password is wrong') bot_status = botstatus.ERROR else: print("That user is not exist in Linkedin.") bot_status = botstatus.ERROR driver.close() return bot_status, lastrun_date, completed_date except Exception as e: bot_status = botstatus.ERROR print(bot_status, ':', e) driver.close() return bot_status, lastrun_date, completed_date
def postconnect_browser(cur, taskrow, msgdetail): email, password = get_user_email_pw(cur, taskrow[6]) contact = get_db(cur, contact_select, (msgdetail[6],)) print('contact:', contact) driver = login_linkedin_withwebdriver(email, password) user_path = "/in/{0}/".format(contact[10]) # go to the user homme userhome = "{0}{1}".format(LINKEDIN_URL, user_path) print('userhome:', userhome) status = True replied_date = None message = "" # click on the send connect try: wait = WebDriverWait(driver, IDLE_INTERVAL_IN_SECONDS) driver.get(userhome) # pv-s-profile-actions--message # connect_btn = driver.find_element_by_class_name("button.pv-s-profile-actions--connect") connect_btn = wait.until(EC.visibility_of_element_located( (By.CSS_SELECTOR, "button.pv-s-profile-actions--connect"))) connect_btn.click() # post the text in the popup box_css = """div#li-modal-container>div.modal-content-wrapper""" # add a note in popup addnote_css = """div#li-modal-container>div.modal-content-wrapper div.send-invite__actions button.button-secondary-large """ btn_addnote = wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, addnote_css))) btn_addnote.click() text_box_css = "{0} {1}".format(box_css, "div.msg-form__compose-area>textarea[name='message']") print('text_box_css:', text_box_css) message_box = wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, text_box_css))) message_box.send_keys(msgdetail[3]) message_box.send_keys(Keys.RETURN) time.sleep(IDLE_INTERVAL_IN_SECONDS) # connect send in popup connect_css = """div#li-modal-container>div.modal-content-wrapper div.send-invite__actions button.button-primary-large """ btn_send = wait.until(EC.visibility_of_element_located( (By.CSS_SELECTOR, connect_css))) btn_send.click() time.sleep(IDLE_INTERVAL_IN_SECONDS) except Exception as err: print('send connect error:', err) status = False # check back id? driver.close() return status
def checkconnect_browser(cur, taskrow, msgdetail): email, password = get_user_email_pw(cur, taskrow[6]) contact = get_db(cur, contact_select, (msgdetail[6],)) print('contact:', contact) driver = login_linkedin_withwebdriver(email, password) user_path = "/in/{0}/".format(contact[10]) # go to the user homme userhome = "{0}{1}".format(LINKEDIN_URL, user_path) print('userhome:', userhome) status = True replied_date = datetime.now() connect = "" message = None # click on the send connect try: wait = WebDriverWait(driver, IDLE_INTERVAL_IN_SECONDS) driver.get(userhome) # pv-s-profile-actions--message # connect_btn = driver.find_element_by_class_name("button.pv-s-profile-actions--connect") message_btn = wait.until(EC.visibility_of_element_located( (By.CSS_SELECTOR, "button.pv-s-profile-actions--message"))) message_btn_wrapper = wait.until(EC.visibility_of_element_located( (By.CSS_SELECTOR, message_btn))) pending_btn = wait.until(EC.visibility_of_element_located( (By.CSS_SELECTOR, "button.pv-s-profile-actions--pending"))) pending_btn_wrapper = wait.until(EC.visibility_of_element_located( (By.CSS_SELECTOR, pending_btn))) inmail_btn = wait.until(EC.visibility_of_element_located( (By.CSS_SELECTOR, "button.pv-s-profile-actions--inmail"))) inmail_btn_wrapper = wait.until(EC.visibility_of_element_located( (By.CSS_SELECTOR, inmail_btn))) if message_btn_wrapper is not None: connect = "connected" if pending_btn_wrapper is not None: connect = "pending" if inmail_btn_wrapper is not None: connect = "ignored" # post the text in the right box box_css = """div.application-outlet>aside.msg-overlay-container div.msg-overlay-conversation-bubble div.msg-overlay-conversation-bubble__content-wrapper div.msg-s-message-list-container """ # textarea.ember-text-area msg-form__textarea" msg_box = wait.until(EC.visibility_of_element_located( (By.CSS_SELECTOR, box_css))) script = "window.scrollBy({x},{y});".format(x=msg_box.location.get('x', 0), y=msg_box.location.get('y', 1000)) print('script:', script) driver.execute_script(script) # print('get_attribute:', msg_box.get_attribute('innerHTML')) msg_list_css = "{0} {1}".format(box_css, "ul.msg-s-message-list>li") print('msg_list_css:', msg_list_css) message_list = driver.find_elements_by_css_selector(msg_list_css) html = message_list[-1].get_attribute('innerHTML') print('message_list:', html) if user_path in html: # found replied replied_date = datetime.now() m = re.search(r'([\d]+):([\d]+)\s+([AP]M)', html) print('time:', m) if m: hr = int(m.group(1)) mi = int(m.group(2)) if m.group(3) == "PM": hr += 12 d = replied_date.day if hr < replied_date.hour: d = d - 1 replied_date = replied_date.replace(minute=mi, hour=hr, day=d) print('replied_date:', replied_date) # get message m = re.search(r'<p class="msg-s-event-listitem__body[^"]*">([^<]+)</p>', html) if m: message = m.group(1).strip() print('message:', message) except Exception as err: print('check connect error:', err) status = False # check back id? driver.close() return status, replied_date, connect, message
def test_message(): driver = login_linkedin_withwebdriver(user_email, user_pw) #time.sleep(10) get_fastmessage(user_email, user_pw, get_cursor(), 77, driver)
def postmessage_browser(cur, taskrow, msgdetail): email, password = get_user_email_pw(cur, taskrow[6]) contact = get_db(cur, contact_select, (msgdetail[6], )) print('contact:', contact) driver = login_linkedin_withwebdriver(email, password) # go to the user homme userhome = "{0}/in/{1}/".format(LINKEDIN_URL, contact[10]) print('userhome:', userhome) status = True # click on the send message try: wait = WebDriverWait(driver, IDLE_INTERVAL_IN_SECONDS) driver.get(userhome) # pv-s-profile-actions--message #message_btn = driver.find_element_by_class_name("pv-s-profile-actions--message") message_btn = wait.until( EC.visibility_of_element_located( (By.CSS_SELECTOR, "button.pv-s-profile-actions--message"))) message_btn.click() # post the text in the right box box_css = """div.application-outlet>aside.msg-overlay-container div.msg-overlay-conversation-bubble div.msg-overlay-conversation-bubble__content-wrapper form.msg-form """ # textarea.ember-text-area msg-form__textarea" form_box = wait.until( EC.visibility_of_element_located((By.CSS_SELECTOR, box_css))) script = "window.scrollBy({x},{y});".format( x=form_box.location.get('x', 0), y=form_box.location.get('y', 1000)) print('script:', script) driver.execute_script(script) #print('get_attribute:', form_box.get_attribute('innerHTML')) text_box_css = "{0} {1}".format( box_css, "div.msg-form__compose-area>textarea[name='message']") print('text_box_css:', text_box_css) message_box = wait.until( EC.visibility_of_element_located((By.CSS_SELECTOR, text_box_css))) message_box.send_keys(msgdetail[3]) message_box.send_keys(Keys.RETURN) time.sleep(IDLE_INTERVAL_IN_SECONDS) # send button_css = "{0} {1}".format(box_css, "footer button.msg-form__send-button") send_btn = wait.until( EC.visibility_of_element_located((By.CSS_SELECTOR, button_css))) send_btn.click() time.sleep(IDLE_INTERVAL_IN_SECONDS) except Exception as err: print('send message error:', err) status = False # check back id? driver.close() return status
def fast_search(email, password, search_data, cur=None, search_id=None, owner_id=None, search_mode=0, limit=750): print('----------', search_data, '----------', search_mode) driver = login_linkedin_withwebdriver(email, password) max_count = limit total_count = 500 lastrun_date = datetime.now() print("==== SEARCH ======") csrf_tocken = get_browser_csrf_tocken(driver) try: if search_mode == 0: time.sleep(5) js = """ var element = document.createElement('div'); element.id = "interceptedSearchResponse"; element.appendChild(document.createTextNode("")); document.body.appendChild(element); var xhttp = new XMLHttpRequest(); xhttp.onreadystatechange = function() { if (this.readyState == 4 && this.status == 200) { document.getElementById("interceptedSearchResponse").innerHTML = this.responseText; } }; xhttp.open("GET", "https://www.linkedin.com/voyager/api/search/cluster?blendedSrpEnabled=true&count=""" + str(max_count) + """&guides=List()&keywords=""" + search_data + """&origin=GLOBAL_SEARCH_HEADER&q=guided&start=0", true); xhttp.setRequestHeader('Csrf-Token', """ + csrf_tocken + """) xhttp.send();""" ############################################################################################################################# driver.execute_script(js) ############################################################################################################################# time.sleep(3) searchListText = driver.find_element_by_id('interceptedSearchResponse').text jsondata = json.loads(searchListText) total_count = jsondata['paging']['total'] print('total_count : ', total_count, searchListText.encode('utf-8')) close_driver(driver, email) return targetcount = max_count if max_count > total_count: targetcount = total_count targetcount = targetcount // 10 + 1 for idx in range(targetcount): ############################################################################################################################# driver.execute_script(""" var element = document.createElement('div'); element.id = "interceptedSearchResponse_""" + str(idx) + """"; element.appendChild(document.createTextNode("")); document.body.appendChild(element); var xhttp = new XMLHttpRequest(); xhttp.onreadystatechange = function() { if (this.readyState == 4 && this.status == 200) { document.getElementById("interceptedSearchResponse_""" + str(idx) + """").innerHTML = this.responseText; } }; xhttp.open("GET", "https://www.linkedin.com/voyager/api/search/cluster?blendedSrpEnabled=true&count=10&guides=List()&keywords=""" + search_data + """&origin=GLOBAL_SEARCH_HEADER&q=guided&start=""" + str( idx * 10) + """", true); xhttp.setRequestHeader('Csrf-Token', """ + csrf_tocken + """) xhttp.send(); """) ############################################################################################################################# time.sleep(10) search_result_cnt = 0 for idx in range(targetcount): conversationListText = driver.find_element_by_id('interceptedSearchResponse_' + str(idx)).text jsondata = json.loads(conversationListText) elementsData = jsondata['elements'][0]['elements'] elementsCount = len(elementsData) for idx2 in range(elementsCount): print(' ----------------------------------------') profileData = elementsData[idx2]['hitInfo']['com.linkedin.voyager.search.SearchProfile'] txt_industry = '' if 'industry' in profileData: txt_industry = profileData['industry'] txt_location = '' if 'location' in profileData: txt_location = profileData['location'] txt_firstname = profileData['miniProfile']['firstName'] txt_lastname = profileData['miniProfile']['lastName'] txt_occupation = profileData['miniProfile']['occupation'] txt_linkedin_id = profileData['miniProfile']['publicIdentifier'] if txt_linkedin_id != 'UNKNOWN': actor_title = "" actor_company = "" if " at " in txt_occupation: title_company = txt_occupation.split(" at ") actor_title = title_company[0] actor_company = title_company[1] else: actor_company = "" actor_title = txt_occupation search_result_cnt += 1 values = (actor_company, txt_industry, txt_location, actor_title, txt_linkedin_id, txt_firstname + ' ' + txt_lastname) get_search_contact_fast(values, cur, owner_id, search_id, search_result_cnt) bot_status = botstatus.DONE elif search_mode == 1: time.sleep(5) driver.get(search_data) print("-------Go to search url-----------") time.sleep(5) total_resultcounts_tag = driver.find_element_by_css_selector( "h3.search-results__total") total_resultcounts = total_resultcounts_tag.text result_counts = total_resultcounts.split(" ") real_counts = result_counts[1] counts = real_counts.replace(",", "") print('counts:', counts) targetcount = max_count if max_count > counts: targetcount = counts targetcount = targetcount // 10 + 1 print('range_count:', targetcount) parse_urls = {} print('parsing url:') for _ in range(targetcount): time.sleep(3) driver.execute_script("window.scrollBy(0, 1000);") time.sleep(3) search_list = driver.find_elements_by_class_name( "search-result__result-link") # print('search_list:', search_list) count = 0 # for search_index in range(len(actor_name_lists)): for tag in search_list: url = tag.get_attribute('href') if url in parse_urls: continue parse_urls[url] = 1 count += 1 if count >= limit: break try: driver.find_element_by_class_name("next").click() except Exception as err1: print('No next:', err1) break get_search_contact_fast_with_urls(parse_urls, driver, cur, owner_id, search_id) """ print('parsing profile:') for count, url in enumerate(parse_urls.keys()): result = parse_connection_link(driver, url) values = result + (owner_id, search_id, botstatus.CONNECT_REQ_N,) print('value insert:', values) add_to_db(cur, search_query, *values) print('count insert:', values) values = (count, search_id, ) add_to_db2(cur, search_update_query, *values) """ bot_status = botstatus.DONE else: time.sleep(2) driver.get(search_data) print("-------Go to sales navigator search url-----------") time.sleep(10) try: total_resultcounts_tag = driver.find_element_by_css_selector( ".TR .spotlight-result-count") except: print("search has not result") values = (0, search_id,) add_to_db2(cur, search_update_query, *values) bot_status = botstatus.DONE driver.close() completed_date = datetime.now() return bot_status, lastrun_date, completed_date total_resultcounts = total_resultcounts_tag.text real_counts = 0 if total_resultcounts[-1:] == 'M': real_counts = float(total_resultcounts[:-1]) * 1000000 elif total_resultcounts[-1:] == 'K': real_counts = float(total_resultcounts[:-1]) * 1000 else: real_counts = int(total_resultcounts) print('counts:', int(real_counts)) if real_counts == 0: print("search has not result") values = (0, search_id,) add_to_db2(cur, search_update_query, *values) bot_status = botstatus.DONE driver.close() completed_date = datetime.now() return bot_status, lastrun_date, completed_date targetcount = max_count if max_count > real_counts: targetcount = real_counts targetcount = targetcount // 25 + 1 print('range_count:', targetcount) print('parsing profile:') for _ in range(targetcount): parse_urls = {} time.sleep(7) search_list = driver.find_elements_by_class_name("member") # for search_index in range(len(actor_name_lists)): count = 0 for tag in search_list: try: search_save_text = tag.find_element_by_class_name( 'save-lead') url = tag.find_element_by_class_name( 'name-link').get_attribute('href') if url in parse_urls: continue parse_urls[url] = 1 count += 1 if count >= limit: break except Exception as err: continue get_search_contact_fast_with_salesurls(parse_urls, driver, cur, owner_id, search_id, count) try: driver.find_element_by_class_name( "next-pagination .pagination-text").click() except Exception as err1: print('No next:', err1) break """ print('parsing profile:') for count, url in enumerate(parse_urls.keys()): result = parse_connection_link_sales(driver, url) # insert into data values = result + (owner_id, search_id, botstatus.CONNECT_REQ_N,) print('value insert:', values) add_to_db(cur, search_query, *values) print('count insert:', values) values = (count, search_id, ) add_to_db2(cur, search_update_query, *values) """ bot_status = botstatus.DONE # completed_date = datetime.now() # return name_list, company_list, title_list, location_list, bot_status, lastrun_date, completed_date except Exception as e: # bot_status = botstatus.ERROR bot_status = botstatus.DONE print("ERROR:", e) driver.close() completed_date = datetime.now() return bot_status, lastrun_date, completed_date