def login_linkedinx(email,
password,
driver=None,
pincode=None,
cur=None,
owner_id=None,
task_id=None,
task_type=None):
print("==== LOGIN =====")
lastrun_date = datetime.now()
completed_date = datetime.now()
login_linkedin_withwebdriver(email, password, driver)
print('get login_linkedin_withwebdriver driver')
wait = WebDriverWait(driver, IDLE_INTERVAL_IN_SECONDS)
# check if user is exist
bot_status = botstatus.DONE
if exist_user(wait):
print("That user is an existing user.")
if check_password_correct(wait):
print('pw is correct')
sleep(1000)
res = check_captcha(driver)
def sale_nav_search(email, password, url, limit=1000):
driver = login_linkedin_withwebdriver(email, password)
max_count = limit
print('url:', url)
# url = url.replace('search?keywords', 'search/results?keywords')
offset = 0
mh = 100
for i in range( limit // mh ):
offset = mh * i
pageurl = url + "&count={count}&start={start}".format(count=mh, start=offset)
#get_search_contact_salesurls(pageurl, driver, None, None, None, i, cookies)
driver.get(pageurl)
time.sleep(5)
"""
soup = BeautifulSoup(driver.page_source, "html.parser")
members = soup.find_all('li', class_='result')
for member in members:
# name and id_
tag_a = member.find('a', class_='name-link')
print( member)
# info
"""
driver.close()
def login_linkedin(email,
password,
pincode=None,
cur=None,
owner_id=None,
task_id=None,
task_type=None):
print("==== LOGIN =====")
lastrun_date = datetime.now()
completed_date = datetime.now()
driver = login_linkedin_withwebdriver(email, password)
try:
wait = WebDriverWait(driver, IDLE_INTERVAL_IN_SECONDS)
# check if user is exist
bot_status = botstatus.DONE
if exist_user(wait):
print("That user is an existing user.")
if check_password_correct(wait):
print('pw is correct')
res = check_captcha(driver)
if res == False:
driver.close()
return bot_status, lastrun_date, completed_date
# pin code verification
res, no_pin_required = pinverify(wait, pincode, cur, owner_id,
task_id, task_type)
if res and no_pin_required:
completed_date = datetime.now()
driver.close()
return bot_status, lastrun_date, completed_date, completed_date
if (no_pin_required == False):
print("Pin to verify!")
if pincode is None:
bot_status = botstatus.PIN_REQUIRED
else:
bot_status = botstatus.PIN_INVALID
else:
print("sucessfull login without pin code verification!")
else:
print('password is wrong')
bot_status = botstatus.ERROR
else:
print("That user is not exist in Linkedin.")
bot_status = botstatus.ERROR
driver.close()
return bot_status, lastrun_date, completed_date
except Exception as e:
bot_status = botstatus.ERROR
print(bot_status, ':', e)
driver.close()
return bot_status, lastrun_date, completed_date
def postconnect_browser(cur, taskrow, msgdetail):
email, password = get_user_email_pw(cur, taskrow[6])
contact = get_db(cur, contact_select, (msgdetail[6],))
print('contact:', contact)
driver = login_linkedin_withwebdriver(email, password)
user_path = "/in/{0}/".format(contact[10])
# go to the user homme
userhome = "{0}{1}".format(LINKEDIN_URL, user_path)
print('userhome:', userhome)
status = True
replied_date = None
message = ""
# click on the send connect
try:
wait = WebDriverWait(driver, IDLE_INTERVAL_IN_SECONDS)
driver.get(userhome)
# pv-s-profile-actions--message
# connect_btn = driver.find_element_by_class_name("button.pv-s-profile-actions--connect")
connect_btn = wait.until(EC.visibility_of_element_located(
(By.CSS_SELECTOR, "button.pv-s-profile-actions--connect")))
connect_btn.click()
# post the text in the popup
box_css = """div#li-modal-container>div.modal-content-wrapper"""
# add a note in popup
addnote_css = """div#li-modal-container>div.modal-content-wrapper div.send-invite__actions button.button-secondary-large """
btn_addnote = wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, addnote_css)))
btn_addnote.click()
text_box_css = "{0} {1}".format(box_css, "div.msg-form__compose-area>textarea[name='message']")
print('text_box_css:', text_box_css)
message_box = wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, text_box_css)))
message_box.send_keys(msgdetail[3])
message_box.send_keys(Keys.RETURN)
time.sleep(IDLE_INTERVAL_IN_SECONDS)
# connect send in popup
connect_css = """div#li-modal-container>div.modal-content-wrapper div.send-invite__actions button.button-primary-large """
btn_send = wait.until(EC.visibility_of_element_located(
(By.CSS_SELECTOR, connect_css)))
btn_send.click()
time.sleep(IDLE_INTERVAL_IN_SECONDS)
except Exception as err:
print('send connect error:', err)
status = False
# check back id?
driver.close()
return status
def checkconnect_browser(cur, taskrow, msgdetail):
email, password = get_user_email_pw(cur, taskrow[6])
contact = get_db(cur, contact_select, (msgdetail[6],))
print('contact:', contact)
driver = login_linkedin_withwebdriver(email, password)
user_path = "/in/{0}/".format(contact[10])
# go to the user homme
userhome = "{0}{1}".format(LINKEDIN_URL, user_path)
print('userhome:', userhome)
status = True
replied_date = datetime.now()
connect = ""
message = None
# click on the send connect
try:
wait = WebDriverWait(driver, IDLE_INTERVAL_IN_SECONDS)
driver.get(userhome)
# pv-s-profile-actions--message
# connect_btn = driver.find_element_by_class_name("button.pv-s-profile-actions--connect")
message_btn = wait.until(EC.visibility_of_element_located(
(By.CSS_SELECTOR, "button.pv-s-profile-actions--message")))
message_btn_wrapper = wait.until(EC.visibility_of_element_located(
(By.CSS_SELECTOR, message_btn)))
pending_btn = wait.until(EC.visibility_of_element_located(
(By.CSS_SELECTOR, "button.pv-s-profile-actions--pending")))
pending_btn_wrapper = wait.until(EC.visibility_of_element_located(
(By.CSS_SELECTOR, pending_btn)))
inmail_btn = wait.until(EC.visibility_of_element_located(
(By.CSS_SELECTOR, "button.pv-s-profile-actions--inmail")))
inmail_btn_wrapper = wait.until(EC.visibility_of_element_located(
(By.CSS_SELECTOR, inmail_btn)))
if message_btn_wrapper is not None:
connect = "connected"
if pending_btn_wrapper is not None:
connect = "pending"
if inmail_btn_wrapper is not None:
connect = "ignored"
# post the text in the right box
box_css = """div.application-outlet>aside.msg-overlay-container div.msg-overlay-conversation-bubble div.msg-overlay-conversation-bubble__content-wrapper div.msg-s-message-list-container """
# textarea.ember-text-area msg-form__textarea"
msg_box = wait.until(EC.visibility_of_element_located(
(By.CSS_SELECTOR, box_css)))
script = "window.scrollBy({x},{y});".format(x=msg_box.location.get('x', 0),
y=msg_box.location.get('y', 1000))
print('script:', script)
driver.execute_script(script)
# print('get_attribute:', msg_box.get_attribute('innerHTML'))
msg_list_css = "{0} {1}".format(box_css, "ul.msg-s-message-list>li")
print('msg_list_css:', msg_list_css)
message_list = driver.find_elements_by_css_selector(msg_list_css)
html = message_list[-1].get_attribute('innerHTML')
print('message_list:', html)
if user_path in html:
# found replied
replied_date = datetime.now()
m = re.search(r'([\d]+):([\d]+)\s+([AP]M)', html)
print('time:', m)
if m:
hr = int(m.group(1))
mi = int(m.group(2))
if m.group(3) == "PM":
hr += 12
d = replied_date.day
if hr < replied_date.hour:
d = d - 1
replied_date = replied_date.replace(minute=mi, hour=hr,
day=d)
print('replied_date:', replied_date)
# get message
m = re.search(r'<p class="msg-s-event-listitem__body[^"]*">([^<]+)</p>', html)
if m:
message = m.group(1).strip()
print('message:', message)
except Exception as err:
print('check connect error:', err)
status = False
# check back id?
driver.close()
return status, replied_date, connect, message
def test_message():
driver = login_linkedin_withwebdriver(user_email, user_pw)
#time.sleep(10)
get_fastmessage(user_email, user_pw, get_cursor(), 77, driver)
def postmessage_browser(cur, taskrow, msgdetail):
email, password = get_user_email_pw(cur, taskrow[6])
contact = get_db(cur, contact_select, (msgdetail[6], ))
print('contact:', contact)
driver = login_linkedin_withwebdriver(email, password)
# go to the user homme
userhome = "{0}/in/{1}/".format(LINKEDIN_URL, contact[10])
print('userhome:', userhome)
status = True
# click on the send message
try:
wait = WebDriverWait(driver, IDLE_INTERVAL_IN_SECONDS)
driver.get(userhome)
# pv-s-profile-actions--message
#message_btn = driver.find_element_by_class_name("pv-s-profile-actions--message")
message_btn = wait.until(
EC.visibility_of_element_located(
(By.CSS_SELECTOR, "button.pv-s-profile-actions--message")))
message_btn.click()
# post the text in the right box
box_css = """div.application-outlet>aside.msg-overlay-container div.msg-overlay-conversation-bubble div.msg-overlay-conversation-bubble__content-wrapper form.msg-form """
# textarea.ember-text-area msg-form__textarea"
form_box = wait.until(
EC.visibility_of_element_located((By.CSS_SELECTOR, box_css)))
script = "window.scrollBy({x},{y});".format(
x=form_box.location.get('x', 0),
y=form_box.location.get('y', 1000))
print('script:', script)
driver.execute_script(script)
#print('get_attribute:', form_box.get_attribute('innerHTML'))
text_box_css = "{0} {1}".format(
box_css, "div.msg-form__compose-area>textarea[name='message']")
print('text_box_css:', text_box_css)
message_box = wait.until(
EC.visibility_of_element_located((By.CSS_SELECTOR, text_box_css)))
message_box.send_keys(msgdetail[3])
message_box.send_keys(Keys.RETURN)
time.sleep(IDLE_INTERVAL_IN_SECONDS)
# send
button_css = "{0} {1}".format(box_css,
"footer button.msg-form__send-button")
send_btn = wait.until(
EC.visibility_of_element_located((By.CSS_SELECTOR, button_css)))
send_btn.click()
time.sleep(IDLE_INTERVAL_IN_SECONDS)
except Exception as err:
print('send message error:', err)
status = False
# check back id?
driver.close()
return status
def fast_search(email, password, search_data, cur=None, search_id=None,
owner_id=None, search_mode=0, limit=750):
print('----------', search_data, '----------', search_mode)
driver = login_linkedin_withwebdriver(email, password)
max_count = limit
total_count = 500
lastrun_date = datetime.now()
print("==== SEARCH ======")
csrf_tocken = get_browser_csrf_tocken(driver)
try:
if search_mode == 0:
time.sleep(5)
js = """
var element = document.createElement('div');
element.id = "interceptedSearchResponse";
element.appendChild(document.createTextNode(""));
document.body.appendChild(element);
var xhttp = new XMLHttpRequest();
xhttp.onreadystatechange = function() {
if (this.readyState == 4 && this.status == 200) {
document.getElementById("interceptedSearchResponse").innerHTML = this.responseText;
}
};
xhttp.open("GET", "https://www.linkedin.com/voyager/api/search/cluster?blendedSrpEnabled=true&count=""" + str(max_count) + """&guides=List()&keywords=""" + search_data + """&origin=GLOBAL_SEARCH_HEADER&q=guided&start=0", true);
xhttp.setRequestHeader('Csrf-Token', """ + csrf_tocken + """)
xhttp.send();"""
#############################################################################################################################
driver.execute_script(js)
#############################################################################################################################
time.sleep(3)
searchListText = driver.find_element_by_id('interceptedSearchResponse').text
jsondata = json.loads(searchListText)
total_count = jsondata['paging']['total']
print('total_count : ', total_count, searchListText.encode('utf-8'))
close_driver(driver, email)
return
targetcount = max_count
if max_count > total_count:
targetcount = total_count
targetcount = targetcount // 10 + 1
for idx in range(targetcount):
#############################################################################################################################
driver.execute_script("""
var element = document.createElement('div');
element.id = "interceptedSearchResponse_""" + str(idx) + """";
element.appendChild(document.createTextNode(""));
document.body.appendChild(element);
var xhttp = new XMLHttpRequest();
xhttp.onreadystatechange = function() {
if (this.readyState == 4 && this.status == 200) {
document.getElementById("interceptedSearchResponse_""" + str(idx) + """").innerHTML = this.responseText;
}
};
xhttp.open("GET", "https://www.linkedin.com/voyager/api/search/cluster?blendedSrpEnabled=true&count=10&guides=List()&keywords=""" + search_data + """&origin=GLOBAL_SEARCH_HEADER&q=guided&start=""" + str(
idx * 10) + """", true);
xhttp.setRequestHeader('Csrf-Token', """ + csrf_tocken + """)
xhttp.send();
""")
#############################################################################################################################
time.sleep(10)
search_result_cnt = 0
for idx in range(targetcount):
conversationListText = driver.find_element_by_id('interceptedSearchResponse_' + str(idx)).text
jsondata = json.loads(conversationListText)
elementsData = jsondata['elements'][0]['elements']
elementsCount = len(elementsData)
for idx2 in range(elementsCount):
print(' ----------------------------------------')
profileData = elementsData[idx2]['hitInfo']['com.linkedin.voyager.search.SearchProfile']
txt_industry = ''
if 'industry' in profileData:
txt_industry = profileData['industry']
txt_location = ''
if 'location' in profileData:
txt_location = profileData['location']
txt_firstname = profileData['miniProfile']['firstName']
txt_lastname = profileData['miniProfile']['lastName']
txt_occupation = profileData['miniProfile']['occupation']
txt_linkedin_id = profileData['miniProfile']['publicIdentifier']
if txt_linkedin_id != 'UNKNOWN':
actor_title = ""
actor_company = ""
if " at " in txt_occupation:
title_company = txt_occupation.split(" at ")
actor_title = title_company[0]
actor_company = title_company[1]
else:
actor_company = ""
actor_title = txt_occupation
search_result_cnt += 1
values = (actor_company, txt_industry, txt_location, actor_title, txt_linkedin_id,
txt_firstname + ' ' + txt_lastname)
get_search_contact_fast(values, cur, owner_id, search_id, search_result_cnt)
bot_status = botstatus.DONE
elif search_mode == 1:
time.sleep(5)
driver.get(search_data)
print("-------Go to search url-----------")
time.sleep(5)
total_resultcounts_tag = driver.find_element_by_css_selector(
"h3.search-results__total")
total_resultcounts = total_resultcounts_tag.text
result_counts = total_resultcounts.split(" ")
real_counts = result_counts[1]
counts = real_counts.replace(",", "")
print('counts:', counts)
targetcount = max_count
if max_count > counts:
targetcount = counts
targetcount = targetcount // 10 + 1
print('range_count:', targetcount)
parse_urls = {}
print('parsing url:')
for _ in range(targetcount):
time.sleep(3)
driver.execute_script("window.scrollBy(0, 1000);")
time.sleep(3)
search_list = driver.find_elements_by_class_name(
"search-result__result-link")
# print('search_list:', search_list)
count = 0
# for search_index in range(len(actor_name_lists)):
for tag in search_list:
url = tag.get_attribute('href')
if url in parse_urls:
continue
parse_urls[url] = 1
count += 1
if count >= limit:
break
try:
driver.find_element_by_class_name("next").click()
except Exception as err1:
print('No next:', err1)
break
get_search_contact_fast_with_urls(parse_urls, driver, cur, owner_id, search_id)
"""
print('parsing profile:')
for count, url in enumerate(parse_urls.keys()):
result = parse_connection_link(driver, url)
values = result + (owner_id, search_id,
botstatus.CONNECT_REQ_N,)
print('value insert:', values)
add_to_db(cur, search_query, *values)
print('count insert:', values)
values = (count, search_id, )
add_to_db2(cur, search_update_query, *values)
"""
bot_status = botstatus.DONE
else:
time.sleep(2)
driver.get(search_data)
print("-------Go to sales navigator search url-----------")
time.sleep(10)
try:
total_resultcounts_tag = driver.find_element_by_css_selector(
".TR .spotlight-result-count")
except:
print("search has not result")
values = (0, search_id,)
add_to_db2(cur, search_update_query, *values)
bot_status = botstatus.DONE
driver.close()
completed_date = datetime.now()
return bot_status, lastrun_date, completed_date
total_resultcounts = total_resultcounts_tag.text
real_counts = 0
if total_resultcounts[-1:] == 'M':
real_counts = float(total_resultcounts[:-1]) * 1000000
elif total_resultcounts[-1:] == 'K':
real_counts = float(total_resultcounts[:-1]) * 1000
else:
real_counts = int(total_resultcounts)
print('counts:', int(real_counts))
if real_counts == 0:
print("search has not result")
values = (0, search_id,)
add_to_db2(cur, search_update_query, *values)
bot_status = botstatus.DONE
driver.close()
completed_date = datetime.now()
return bot_status, lastrun_date, completed_date
targetcount = max_count
if max_count > real_counts:
targetcount = real_counts
targetcount = targetcount // 25 + 1
print('range_count:', targetcount)
print('parsing profile:')
for _ in range(targetcount):
parse_urls = {}
time.sleep(7)
search_list = driver.find_elements_by_class_name("member")
# for search_index in range(len(actor_name_lists)):
count = 0
for tag in search_list:
try:
search_save_text = tag.find_element_by_class_name(
'save-lead')
url = tag.find_element_by_class_name(
'name-link').get_attribute('href')
if url in parse_urls:
continue
parse_urls[url] = 1
count += 1
if count >= limit:
break
except Exception as err:
continue
get_search_contact_fast_with_salesurls(parse_urls, driver,
cur, owner_id,
search_id, count)
try:
driver.find_element_by_class_name(
"next-pagination .pagination-text").click()
except Exception as err1:
print('No next:', err1)
break
"""
print('parsing profile:')
for count, url in enumerate(parse_urls.keys()):
result = parse_connection_link_sales(driver, url)
# insert into data
values = result + (owner_id, search_id,
botstatus.CONNECT_REQ_N,)
print('value insert:', values)
add_to_db(cur, search_query, *values)
print('count insert:', values)
values = (count, search_id, )
add_to_db2(cur, search_update_query, *values)
"""
bot_status = botstatus.DONE
# completed_date = datetime.now()
# return name_list, company_list, title_list, location_list, bot_status, lastrun_date, completed_date
except Exception as e:
# bot_status = botstatus.ERROR
bot_status = botstatus.DONE
print("ERROR:", e)
driver.close()
completed_date = datetime.now()
return bot_status, lastrun_date, completed_date