class Message(): def __init__(self, username, password, consumer_key, consumer_secret, access_token, access_secret): self.launcher = Launcher(username, password, consumer_key, consumer_secret, access_token, access_secret) self.api = self.launcher.api() self.es = Es_twitter() self.list = [] self.root_uid = self.api.me() self.update_time = int(time.time()) def get_message(self): for each in self.api.direct_messages(): print each.sender._json content = each.text sender_screen_name = each.sender._json['name'] sender_user_name = each.sender._json['user_name'] sender_id = each.sender._json['id'] timestamp = int(time.mktime(each.created_at.timetuple())) photo_url = each.sender._json['profile_image_url_https'] item = { 'uid': sender_id, 'photo_url': photo_url, 'user_name': sender_screen_name, 'nick_name': sender_user_name, 'timestamp': timestamp, 'text': content, 'update_time': self.update_time } self.list.append(item) return self.list def save(self, indexName, typeName, list): self.es.executeES(indexName, typeName, list)
class Message(): def __init__(self, username, password): self.launcher = Launcher(username, password) self.api = self.launcher.api() self.es = Es_twitter() self.list = [] self.root_uid = self.api.me() def get_message(self): for each in self.api.direct_messages(): content = each.text sender_screen_name = each.sender._json['screen_name'] sender_id = each.sender._json['id'] timestamp = int(time.mktime(each.created_at.timetuple())) mid = each.id item = { 'nick_name': sender_screen_name, 'uid': sender_id, 'text': content, 'timestamp': timestamp, 'root_uid': self.root_uid, 'mid': mid } self.list.append(item) return self.list def save(self, indexName, typeName, list): self.es.executeES(indexName, typeName, list)
class Like(): def __init__(self, username, password, consumer_key, consumer_secret, access_token, access_secret): self.launcher = Launcher(username, password, consumer_key, consumer_secret, access_token, access_secret) self.driver = self.launcher.login() self.es = Es_twitter() self.api = self.launcher.api() self.driver.get('https://twitter.com/i/notifications') time.sleep(2) self.lis = self.driver.find_elements_by_xpath( '//li[@data-item-type="activity"]') self.list = [] self.update_time = int(time.time()) def get_like(self): try: for li in self.lis: type = li.get_attribute('data-component-context') if type == "favorite_activity": user_name = li.find_element_by_xpath( './div/div/div/div[2]/div[1]/a/strong').text screen_name = li.find_element_by_xpath( './div/div/div/div[2]/div[1]/a').get_attribute( 'href').replace('https:twitter.com/', '') timestamp = li.find_element_by_xpath( './div/div/div/div[2]/div[1]/div[1]/div/span' ).get_attribute('data-time') user_id = li.find_element_by_xpath( './div/div/div/div[2]/div[1]/a').get_attribute( 'data-user-id') root_user_id = li.find_element_by_xpath( './div/div/div/div[2]/div[2]/div/div/div' ).get_attribute('data-user-id') root_content = li.find_element_by_xpath( './div/div/div/div[2]/div[2]/div/div/div/div/div/div[2]' ).text mid = li.get_attribute('data-item-id') photo_url = li.find_element_by_xpath( './div/div/div/div[2]//img').get_attribute('src') item = { 'uid': user_id, 'photo_url': photo_url, 'user_name': screen_name, 'nick_name': user_name, 'timestamp': int(timestamp), 'text': root_content, 'update_time': self.update_time, 'root_text': root_content, 'root_mid': mid } self.list.append(item) finally: self.driver.close() return self.list def save(self, indexName, typeName, list): self.es.executeES(indexName, typeName, list)
class Follower(): def __init__(self, username, password, current_ts, consumer_key, consumer_secret, access_token, access_secret, *args): self.launcher = Launcher(username, password, consumer_key, consumer_secret, access_token, access_secret) self.api = self.launcher.api() self.es = Es_twitter() self.list = [] self.update_time = int(time.time()) try: self.uid = args[0] except Exception as e: print "no uid" def get_follower(self): try: for each in self.api.followers(self.uid): nick_name = each.name print 'nick_name:', nick_name user_name = each.screen_name print 'user_name:', user_name id = each.id print 'uid:', id photo_url = each.profile_image_url_https print 'photo_url:', photo_url profile_url = 'https://twitter.com/' + screen_name print 'profile_url:', profile_url item = { 'user_name':user_name, 'nick_name':nick_name, 'uid':id, 'photo_url':photo_url, 'profile_url':profile_url } self.list.append(item) except Exception as e: for each in self.api.followers(): name = each.name screen_name = each.screen_name id = each.id photo_url = each.profile_image_url_https profile_url = 'https://twitter.com/' + screen_name item = { 'user_name':name, 'nick_name':screen_name, 'uid':id, 'photo_url':photo_url, 'profile_url':profile_url, 'update_time':self.update_time, } self.list.append(item) return self.list def save(self, indexName, typeName, follower_list): self.es.executeES(indexName, typeName, follower_list)
class Like(): def __init__(self, username, password): self.launcher = Launcher(username, password) self.driver = self.launcher.login() self.es = Es_twitter() self.api = self.launcher.api() self.driver.get('https://twitter.com/i/notifications') time.sleep(2) self.lis = self.driver.find_elements_by_xpath( '//li[@data-item-type="activity"]') self.list = [] def get_like(self): for li in self.lis: type = li.get_attribute('data-component-context') if type == "favorite_activity": user_name = li.find_element_by_xpath( './div/div/div/div[2]/div[1]/a/strong').text timestamp = li.find_element_by_xpath( './div/div/div/div[2]/div[1]/div[1]/div/span' ).get_attribute('data-time') user_id = li.find_element_by_xpath( './div/div/div/div[2]/div[1]/a').get_attribute( 'data-user-id') root_user_name = li.find_element_by_xpath( './div/div/div/div[2]/div[2]/div/div/div/div/div/div[1]/b' ).text root_user_screen_name = li.find_element_by_xpath( './div/div/div/div[2]/div[2]/div/div/div/div/div/div[1]/span[3]/b' ).text root_user_id = li.find_element_by_xpath( './div/div/div/div[2]/div[2]/div/div/div').get_attribute( 'data-user-id') root_content = li.find_element_by_xpath( './div/div/div/div[2]/div[2]/div/div/div/div/div/div[2]' ).text mid = li.get_attribute('data-item-id') item = { 'nick_name': user_name, 'timestamp': int(timestamp), 'uid': user_id, 'root_user_name': root_user_name, 'root_nick_name': root_user_screen_name, 'root_uid': root_user_id, 'root_text': root_content, 'mid': mid } self.list.append(item) return self.list def save(self, indexName, typeName, list): self.es.executeES(indexName, typeName, list)
class Userinfo(): def __init__(self, username, password): self.launcher = Launcher(username, password) def getUserinfo(self): driver, display = self.launcher.login() time.sleep(2) driver.find_element_by_xpath( '//a[@data-component-context="home_nav"]').click() time.sleep(2) screen_name_url = driver.find_element_by_xpath( '//div[@class="DashboardProfileCard-content"]//a').get_attribute( 'href') driver.get(screen_name_url) api = self.launcher.api() time.sleep(1) screen_name = driver.find_element_by_xpath( '//div[@class="ProfileHeaderCard"]/h2//b').text uid = api.get_user(screen_name).id try: description = api.get_user(screen_name).description except: description = None try: location = api.get_user(screen_name).location except: location = None today = int(time.strftime('%Y', time.localtime(int(time.time())))) try: birth = driver.find_element_by_xpath( '//span[@class="ProfileHeaderCard-birthdateText u-dir"]/span' ).text except: birth = False pattern = re.compile(u'(\d+)年') pattern1 = re.compile(', (\d+)') if birth: try: birthday = int(re.findall(pattern, birth)[0]) except: birthday = int(re.findall(pattern1, birth)[0]) age = today - birthday else: age = None dict = { 'uid': uid, 'desccription': description, 'location': location, 'age': age } driver.quit() display.popen.kill() return dict
class At(): def __init__(self, username, password, consumer_key, consumer_secret, access_token, access_secret): self.launcher = Launcher(username, password, consumer_key, consumer_secret, access_token, access_secret) self.api = self.launcher.api() self.es = Es_twitter() self.list = [] self.update_time = int(time.time()) def get_mention(self): for each in self.api.mentions_timeline(): try: user_screen_name = each.author.screen_name print "user_screen_name", user_screen_name user_name = each.author.name print "user_name", user_name user_id = each.author.id print "user_id", user_id text = each.text print "text", text user_mention_screen_name = each.entities['user_mentions'][0][ 'screen_name'] print "user_mention_screen_name", user_mention_screen_name user_mention_name = each.entities['user_mentions'][0]['name'] print "user_mention_name", user_mention_name user_mention_id = each.entities['user_mentions'][0]['id'] print "user_mention_id", user_mention_id timestamp = int(time.mktime(each.created_at.timetuple())) print "timestamp", timestamp mid = each.id print "mid", mid photo_url = each.author.profile_image_url_https print "photo_url", photo_url item = { 'uid': user_id, 'photo_url': photo_url, 'user_name': user_screen_name, 'nick_name': user_name, 'mid': mid, 'timestamp': timestamp, 'text': text, 'update_time': self.update_time } self.list.append(item) except: pass return self.list def save(self, indexName, typeName, list): self.es.executeES(indexName, typeName, list)
class Follower(): def __init__(self, username, password, current_ts, consumer_key, consumer_secret, access_token, access_secret, *args): self.launcher = Launcher(username, password, consumer_key, consumer_secret, access_token, access_secret) self.api = self.launcher.api() self.es = Es_twitter() self.list = [] self.update_time = int(time.time()) try: self.uid = args[0] except Exception as e: print "no uid" def get_follow(self): try: for each in self.api.friends_ids(self.uid): id = each name = self.api.get_user(each).name screen_name = self.api.get_user(each).screen_name photo_url = self.api.get_user(each).profile_image_url_https profile_url = 'https://twitter.com/' + screen_name item = { 'user_name': name, 'nick_name': screen_name, 'uid': id, 'photo_url': photo_url, 'profile_url': profile_url } self.list.append(item) except Exception as e: for each in self.api.friends_ids(): id = each name = self.api.get_user(each).name screen_name = self.api.get_user(each).screen_name photo_url = self.api.get_user(each).profile_image_url_https profile_url = 'https://twitter.com/' + screen_name item = { 'user_name': name, 'nick_name': screen_name, 'uid': id, 'photo_url': photo_url, 'profile_url': profile_url, 'update_time': self.update_time, } self.list.append(item) self.launcher.display.popen.kill() return self.list def save(self, indexName, typeName, list): self.es.executeES(indexName, typeName, list)
class Userinfo(): def __init__(self, username, password, consumer_key, consumer_secret, access_token, access_secret): self.launcher = Launcher(username, password, consumer_key, consumer_secret, access_token, access_secret) def getUserinfo(self): #driver = self.launcher.login_mobile() #time.sleep(2) #driver.find_element_by_xpath('//nav[@aria-label="Primary"]/a[@aria-label="Home timeline"]/div/svg/g/path').click() #time.sleep(2) #screen_name_url = driver.find_element_by_xpath('//div[@class="DashboardProfileCard-content"]//a').get_attribute('href') #driver.get(screen_name_url) api = self.launcher.api() user = api.me() #time.sleep(1) #screen_name = driver.find_element_by_xpath('//div[@class="ProfileHeaderCard"]/h2//b').text uid = user.id_str screen_name = user.screen_name #uid = api.get_user(screen_name).id try: description = user.description except: description = '' try: location = user.location except: location = '' #today = int(time.strftime('%Y',time.localtime(int(time.time())))) #try: # birth = driver.find_element_by_xpath('//span[@class="ProfileHeaderCard-birthdateText u-dir"]/span').text #except: # birth = False #pattern = re.compile(u'(\d+)年') #pattern1 = re.compile(', (\d+)') #if birth: # try: # birthday = int(re.findall(pattern,birth)[0]) # except: # birthday = int(re.findall(pattern1,birth)[0]) # age = today - birthday #else: # age = None dict = {'uid': uid, 'desccription': description, 'location': location} return dict
class Follower(): def __init__(self, username, password, current_ts, *args): self.launcher = Launcher(username, password) self.api = self.launcher.api() self.es = Es_twitter() self.update_time = current_ts self.list = [] try: self.uid = args[0] except Exception as e: print "no uid" def get_follow(self): try: for each in self.api.friends_ids(self.uid): id = each name = self.api.get_user(each).name screen_name = self.api.get_user(each).screen_name root_uid = self.api.me().id item = { 'user_name':name, 'nick_name':screen_name, 'uid':id, 'update_time':self.update_time, 'root_uid':root_uid } self.list.append(item) except Exception as e: for each in self.api.friends_ids(): id = each name = self.api.get_user(each).name screen_name = self.api.get_user(each).screen_name root_uid = self.api.me().id item = { 'user_name':name, 'nick_name':screen_name, 'uid':id, 'update_time':self.update_time, 'root_uid':root_uid } self.list.append(item) return self.list def save(self, indexName, typeName, list): self.es.executeES(indexName, typeName, list)
class At(): def __init__(self, username, password): self.launcher = Launcher(username, password) self.api = self.launcher.api() self.es = Es_twitter() self.list = [] def get_mention(self): for each in self.api.mentions_timeline(): try: user_screen_name = each.author.screen_name user_name = each.author.name user_id = each.author.id text = each.text user_mention_screen_name = each.entities['user_mentions'][0][ 'screen_name'] user_mention_name = each.entities['user_mentions'][0]['name'] user_mention_id = each.entities['user_mentions'][0]['id'] timestamp = int(time.mktime(each.created_at.timetuple())) mid = each.id item = { 'user_name': user_screen_name, 'nick_name': user_name, 'uid': user_id, 'text': text, 'user_mention_screen_name': user_mention_screen_name, 'user_mention_name': user_mention_name, 'user_mention_id': user_mention_id, 'root_uid': user_mention_id, 'timestamp': timestamp, 'mid': mid } self.list.append(item) except: pass return self.list def save(self, indexName, typeName, list): self.es.executeES(indexName, typeName, list)
class Follower(): def __init__(self, username, password, current_ts, consumer_key, consumer_secret, access_token, access_secret, *args): self.launcher = Launcher(username, password, consumer_key, consumer_secret, access_token, access_secret) self.api = self.launcher.api() self.es = Es_twitter() self.list = [] self.update_time = int(time.time()) try: self.uid = args[0] except Exception as e: print "no uid" def get_follow(self): try: for each in self.api.friends_ids(self.uid): id = each print id name = self.api.get_user(each).name print name screen_name = self.api.get_user(each).screen_name print screen_name photo_url = self.api.get_user(each).profile_image_url_https print phtot_url
class Operation(): def __init__(self, username, password, consumer_key, consumer_secret, access_token, access_secret): self.launcher = Launcher(username, password, consumer_key, consumer_secret, access_token, access_secret) self.api = self.launcher.api() self.list = [] def publish(self, text): try: self.api.update_status(text) time.sleep(2) return [True, ' '] except Exception as e: return [False, e] finally: self.launcher.display.popen.kill() #API.update_with_media(filename[, status][, in_reply_to_status_id][, auto_populate_reply_metadata][, lat][, long][, source][, place_id][, file]) def publish_with_media(self, filename, text): try: self.api.update_with_media(filename, text) time.sleep(2) return [True, ' '] except Exception as e: return [False, e] finally: self.launcher.display.popen.kill() def mention(self, text): #text = '@lvleilei1 test' try: self.api.update_status(text) time.sleep(2) return [True, ' '] except Exception as e: return [False, e] finally: self.launcher.display.popen.kill() def message(self, uid, text): try: print self.api.send_direct_message(uid, text=text) time.sleep(2) return [True, ' '] except Exception as e: return [False, e] finally: self.launcher.display.popen.kill() def follow(self, uid): try: self.api.create_friendship(uid) time.sleep(2) return [True, ' '] except Exception as e: return [False, e] finally: self.launcher.display.popen.kill() def destroy_friendship(self, uid): try: self.api.destroy_friendship(uid) time.sleep(2) return [True, ' '] except Exception as e: return [False, e] finally: self.launcher.display.popen.kill() def do_retweet(self, tid): try: self.api.retweet(tid) time.sleep(2) return [True, ' '] except Exception as e: return [False, e] finally: self.launcher.display.popen.kill() def do_retweet_text(self, uid, tid, text): try: driver = self.launcher.login() screen_name = self.launcher.get_user(uid) post_url = 'https://twitter.com/' + screen_name + '/status/' + tid driver.get(post_url) time.sleep(3) current_url = driver.current_url pattern = re.compile('status/(\d+)') primary_id = ''.join(re.findall(pattern, current_url)).strip() driver.find_element_by_xpath( '//button[@aria-describedby="profile-tweet-action-retweet-count-aria-%s"]' % primary_id).click() time.sleep(3) driver.find_element_by_xpath( '//div[@id="retweet-with-comment"]').click() driver.find_element_by_xpath( '//div[@id="retweet-with-comment"]').send_keys(text) driver.find_element_by_xpath( '//button[@class="EdgeButton EdgeButton--primary retweet-action"]' ).click() time.sleep(2) return [True, ' '] except Exception as e: return [False, e] finally: driver.quit() self.launcher.display.popen.kill() def do_favourite(self, tid): try: self.api.create_favorite(tid) time.sleep(2) return [True, ' '] except Exception as e: return [False, e] finally: self.launcher.display.popen.kill() def do_comment(self, uid, tid, text): try: driver = self.launcher.login() screen_name = self.launcher.get_user(uid) post_url = 'https://twitter.com/' + screen_name + '/status/' + tid driver.get(post_url) time.sleep(1) current_url = driver.current_url pattern = re.compile('status/(\d+)') primary_id = ''.join(re.findall(pattern, current_url)).strip() driver.find_element_by_xpath('//div[@id="tweet-box-reply-to-%s"]' % primary_id).click() driver.find_element_by_xpath('//div[@id="tweet-box-reply-to-%s"]' % primary_id).send_keys(text) time.sleep(1) driver.find_element_by_xpath( '//button[@class="tweet-action EdgeButton EdgeButton--primary js-tweet-btn"]' ).click() time.sleep(2) return [True, ' '] except Exception as e: return [False, e] finally: driver.quit() self.launcher.display.popen.kill()
class Like(): def __init__(self, username, password, consumer_key, consumer_secret, access_token, access_secret): self.launcher = Launcher(username, password, consumer_key, consumer_secret, access_token, access_secret) #self.driver = self.launcher.login() self.es = Es_twitter() self.api = self.launcher.api() #self.driver.get('https://twitter.com/i/notifications') time.sleep(2) #self.lis = self.driver.find_elements_by_xpath('//li[@data-item-type="activity"]') self.like_list = [] self.update_time = int(time.time()) def date2timestamp(self, date): format_time = time.strptime(date, '%a %b %d %H:%M:%S +0000 %Y') timestamp = int(time.mktime(format_time)) return timestamp def get_like(self): favorites = self.api.favorites() #for li in self.lis: for favorite in favorites: #type = li.get_attribute('data-component-context') #if type == "favorite_activity": # user_name = li.find_element_by_xpath('./div/div/div/div[2]/div[1]/a/strong').text user_name = favorite._json['user']['screen_name'] print "user_name", user_name #screen_name = li.find_element_by_xpath('./div/div/div/div[2]/div[1]/a').get_attribute('href').replace('https:twitter.com/','') nick_name = favorite._json['user']['name'] print "nick_name", nick_name #timestamp = li.find_element_by_xpath('./div/div/div/div[2]/div[1]/div[1]/div/span').get_attribute('data-time') timestamp = self.date2timestamp( favorite._json['user']['created_at']) print "timestamp", timestamp #user_id = li.find_element_by_xpath('./div/div/div/div[2]/div[1]/a').get_attribute('data-user-id') uid = favorite._json['user']['id'] print "uid", uid # root_user_id = li.find_element_by_xpath('./div/div/div/div[2]/div[2]/div/div/div').get_attribute('data-user-id') #print "root_user_id", root_user_id # root_content = li.find_element_by_xpath('./div/div/div/div[2]/div[2]/div/div/div/div/div/div[2]').text root_text = favorite.text print "root_text", root_text # mid = li.get_attribute('data-item-id') root_mid = favorite.id print "root_mid", root_mid # photo_url = li.find_element_by_xpath('./div/div/div/div[2]//img').get_attribute('src') photo_url = favorite._json['user']['profile_image_url_https'] print "photo_url", photo_url item = { 'uid': uid, 'photo_url': photo_url, 'user_name': user_name, 'nick_name': nick_name, 'timestamp': int(timestamp), 'root_text': root_text, 'update_time': self.update_time, #'root_text':root_content, 'root_mid': root_mid } self.like_list.append(item) #self.driver.quit() return self.like_list def save(self, indexName, typeName, like_list): self.es.executeES(indexName, typeName, self.like_list)
class Share(): def __init__(self, username, password, consumer_key, consumer_secret, access_token, access_secret): self.launcher = Launcher(username, password, consumer_key, consumer_secret, access_token, access_secret) self.api = self.launcher.api() self.driver, self.display = self.launcher.login() self.driver.get('https://twitter.com/i/notifications') self.es = Es_twitter() self.lis = self.driver.find_elements_by_xpath( '//li[@data-item-type="activity"]') self.list = [] self.update_time = int(time.time()) def get_share(self): try: root_uid = self.api.me().id for li in self.lis: try: type = li.get_attribute('data-component-context') if type == "quote_activity" or type == "retweet_activity": try: mid = li.get_attribute('data-item-id') print "mid", mid user_name = li.find_element_by_xpath( './div/div[2]/div[1]/a/span[1]').text print "user_name", user_name screen_name = li.find_element_by_xpath( './div/div[2]/div[1]/a').get_attribute( 'href').replace('https:twitter.com/', '') print "screen_name", screen_name photo_url = li.find_element_by_xpath( './div/div[2]//img').get_attribute('src') print "photo_url", photo_url user_id = li.find_element_by_xpath( './div/div[2]/div[1]/a').get_attribute( 'data-user-id') print "user_id", user_id timestamp = int( li.find_element_by_xpath( './div/div[2]/div[1]/small/a/span'). get_attribute('data-time')) print "timestamp", timestamp content = li.find_element_by_xpath( './div/div[2]/div[2]/p').text print "content", content root_user_screen_name = li.find_element_by_xpath( './div/div[2]/div[3]/div/div/div[1]/div[1]/div[1]/span[3]/b' ).text print "root_user_screen_name", root_user_screen_name root_content = li.find_element_by_xpath( './div/div[2]/div[3]/div/div[1]/div[1]/div[1]/div[2]' ).text print "root_content", root_content except: mid = li.get_attribute('data-item-id') user_name = li.find_element_by_xpath( './div/div/div/div[2]/div/a').text screen_name = li.find_element_by_xpath( './div/div/div/div[2]/div/a').get_attribute( 'href').replace('https:twitter.com/', '') photo_url = li.find_element_by_xpath( './div/div/div/div[2]//img').get_attribute( 'src') user_id = li.find_element_by_xpath( './div/div/div/div[2]/div/a').get_attribute( 'data-user-id') timestamp = int( li.find_element_by_xpath( './div/div/div/div[2]/div/div/div/span'). get_attribute('data-time')) content = 'None' root_uid = li.find_element_by_xpath( './div/div/div/div[2]/div[2]/div/div/div' ).get_attribute('data-user-id') root_user_screen_name = li.find_element_by_xpath( './div/div/div/div[2]/div[2]/div/div/div' ).get_attribute('data-screen-name') root_content = li.find_element_by_xpath( './div/div/div/div[2]/div[2]/div/div/div/div/div/div[2]' ).text item = { 'uid': user_id, 'photo_url': photo_url, 'user_name': screen_name, 'nick_name': user_name, 'mid': mid, 'timestamp': timestamp, 'text': content, 'update_time': self.update_time, 'root_text': root_content, 'root_mid': mid } self.list.append(item) except: pass finally: self.driver.quit() self.display.popen.kill() return self.list def save(self, indexName, typeName, list): self.es.executeES(indexName, typeName, list)
class Share(): def __init__(self, username, password): self.launcher = Launcher(username, password) self.api = self.launcher.api() self.driver = self.launcher.login() self.driver.get('https://twitter.com/i/notifications') self.es = Es_twitter() self.lis = self.driver.find_elements_by_xpath( '//li[@data-item-type="activity"]') self.list = [] def get_share(self): root_uid = self.api.me().id for li in self.lis: try: type = li.get_attribute('data-component-context') if type == "quote_activity" or type == "retweet_activity": try: mid = li.get_attribute('data-item-id') user_name = li.find_element_by_xpath( './div/div[2]/div[1]/a/span[1]').text #user_screen_name = li.find_element_by_xpath('./div/div[2]/div[1]/a/span[2]').text user_id = li.find_element_by_xpath( './div/div[2]/div[1]/a').get_attribute( 'data-user-id') timestamp = int( li.find_element_by_xpath( './div/div[2]/div[1]/small/a/span'). get_attribute('data-time')) content = li.find_element_by_xpath( './div/div[2]/div[2]/p').text #root_user_name = li.find_element_by_xpath('./div/div[2]/div[3]/div/div/div[1]/div[1]/div[1]/b').text root_user_screen_name = li.find_element_by_xpath( './div/div[2]/div[3]/div/div/div[1]/div[1]/div[1]/span[3]/b' ).text root_content = li.find_element_by_xpath( './div/div[2]/div[3]/div/div[1]/div[1]/div[1]/div[2]' ).text except: mid = li.get_attribute('data-item-id') user_name = li.find_element_by_xpath( './div/div/div/div[2]/div/a').text user_id = li.find_element_by_xpath( './div/div/div/div[2]/div/a').get_attribute( 'data-user-id') timestamp = int( li.find_element_by_xpath( './div/div/div/div[2]/div/div/div/span'). get_attribute('data-time')) content = 'None' root_uid = li.find_element_by_xpath( './div/div/div/div[2]/div[2]/div/div/div' ).get_attribute('data-user-id') root_user_screen_name = li.find_element_by_xpath( './div/div/div/div[2]/div[2]/div/div/div' ).get_attribute('data-screen-name') root_content = li.find_element_by_xpath( './div/div/div/div[2]/div[2]/div/div/div/div/div/div[2]' ).text item = { 'user_name': user_name, #'nick_name':user_screen_name, 'uid': user_id, 'timestamp': timestamp, 'text': content, #'root_user_name':root_user_name, 'root_nick_name': root_user_screen_name, 'root_uid': root_uid, 'root_text': root_content, 'mid': mid } self.list.append(item) except: pass return self.list def save(self, indexName, typeName, list): self.es.executeES(indexName, typeName, list)