def _clickRetweet(self, tweetbox): """ Retweets a tweet @param tweetbox {WebElement} Selenium element of tweet """ css = '.' + \ 'ProfileTweet-actionButton ProfileTweet-follow-button js-tooltip'.replace( ' ', ',') buttons = tweetbox.find_elements(By.CSS_SELECTOR, css) button_num = 0 for button in buttons: if ("Retweet" == button.text.split('\n')[0]): button.click() sleep(0.5) css = 't1-form tweet-form RetweetDialog-tweetForm isWithoutComment condensed' css = '.' + css.replace(' ', '.') retweet_box = self.driver.find_element(By.CSS_SELECTOR, css) css = '.btn.primary-btn.retweet-action' sleep(0.5) retweet_box.find_element(By.CSS_SELECTOR, css).click() self.logger.debug('Retweeted ' + self.tweetinfo['handle']) self.tweetinfo['type'] = 'rt' database_commands.insertTweet(self.tweetinfo) sleep(0.5) try: css = 't1-form tweet-form RetweetDialog-tweetForm isWithoutComment condensed' css = 'Icon Icon--close Icon--medium dismissIcon Icon--close Icon--medium dismiss' css = 'modal-btn modal-close js-close' css = '.' + css.replace(' ', '.') exit = self.driver.find_element(By.CSS_SELECTOR, css) exit.click() return True self.logger.debug('*' * 30) self.logger.debug('Exited the Retweet') self.logger.debug('*' * 30) except: return True
def collectTweets(self, twitterhandle): """ Collects all/latest tweets Saves tweets to the database. @param twitterhandle {String} name of users twitter handle """ if not self.signedIn: self.signin() if '@' in twitterhandle: twitterhandle = twitterhandle.split('@')[1] # Store User Handle handles = database_commands.getHandler(twitterhandle) if len(handles) < 1: self.saveTwitterHandle(twitterhandle) sleep(1) self.liveSearch('from:' + twitterhandle) boxInd = 0 lastNumBoxes = 0 self.tweetboxes = self._loadAllTweets(numTimes=1) numBoxes = len(self.tweetboxes) inserted = True while lastNumBoxes != numBoxes and inserted: while boxInd < len(self.tweetboxes) and inserted: tweetbox = self.tweetboxes[boxInd] try: tweet = self._getTweetStats(tweetbox) tweet['handle'] = twitterhandle inserted = database_commands.insertTweet(tweet) except: inserted = True boxInd += 1 if inserted: self.tweetboxes = self._loadAllTweets(numTimes=5) lastNumBoxes = numBoxes numBoxes = len(self.tweetboxes) self.logger.info('Completed ' + str(boxInd) + ' tweets, loaded ' + str(numBoxes-lastNumBoxes) + ' more tweets for ' + twitterhandle) self.logger.info( 'Inserted ' + str(boxInd - 1) + ' tweets for ' + twitterhandle)
def collectAllTweets(self, twitterhandle): """ Collects all tweets Saves tweets to the database. Continues searching until it finds no *new* tweets for 6 consecutive months. @param twitterhandle {String} name of users twitter handle """ if not self.signedIn: self.signin() if '@' in twitterhandle: twitterhandle = twitterhandle.split('@')[1] # Store User Handle handles = database_commands.getHandler(twitterhandle) if len(handles) < 1: self.saveTwitterHandle(twitterhandle) sleep(1) numZeros = 0 totalInserted = 0 twitterDates = utils.allTwitterDates drySpan = 6 if self.phantom: twitterDates = utils.allTwitterDatesByDay drySpan = 180 for i in range(len(twitterDates)-2,-1,-1): try: cmd = 'from:' + twitterhandle + ' since:' + twitterDates[i] + ' until:' + twitterDates[i+1] self.logger.info('searching: "' + cmd + "'") self.liveSearch(cmd) boxInd = 0 lastNumBoxes = 0 self.tweetboxes = self._loadAllTweets(numTimes=5) numBoxes = len(self.tweetboxes) numInserted = 0 while lastNumBoxes != numBoxes: while boxInd < len(self.tweetboxes): tweetbox = self.tweetboxes[boxInd] inserted = False try: tweet = self._getTweetStats(tweetbox) tweet['handle'] = twitterhandle inserted = database_commands.insertTweet(tweet,insertDuplicates=False) if inserted: numInserted += 1 except: pass boxInd += 1 self.tweetboxes = self._loadAllTweets(numTimes=5) lastNumBoxes = numBoxes numBoxes = len(self.tweetboxes) self.logger.info('Inserted ' + str(numInserted) + ' tweets, loaded ' + str(numBoxes-lastNumBoxes) + ' more tweets for ' + twitterhandle) if numInserted == 0: numZeros += 1 else: numZeros = 0 if numZeros > drySpan: break totalInserted += numInserted self.logger.info('Inserted ' + str(totalInserted) + ' TOTAL tweets for ' + twitterhandle + '. Zeros in-a-row: ' + str(numZeros)) except Exception as e: traceback.print_exc() traceback.print_stack() self.logger.error('Error scrolling through tweets!') self.logger.error(e)