def getPosts(user,clean=True,maxPosts=100): try: userName = user.facebookId args = {} graph = GraphAPI(at) profile = graph.get_object(userName) posts = graph.get_connections(profile['id'], 'posts', **args) statuses = [] lastUntil = '' for s in posts['paging']['next'].split('&'): if 'limit=' in s: args['limit'] = s.split('=')[1] if 'until=' in s: args['until'] = s.split('=')[1] while True: isBreak = False for data in posts['data']: if 'message' not in data: continue smessage = str(data['message'].encode("utf-8")).replace('\\n', ' ').replace('\\r', '')[2:-1] if clean: smessage = re.sub(r'https?:\/\/[^\s]*','',smessage, flags=re.MULTILINE) smessage = re.sub(r'\\x..','',smessage, flags=re.MULTILINE) smessage = re.sub(r'&','and',smessage, flags=re.IGNORECASE) smessage = re.sub(r'[\s\t]+',' ',smessage, flags=re.IGNORECASE) smessage.replace('#','') if validPost(smessage): st = Status() st.statusText = smessage st.id = data['id'] statuses.append(st) if len(statuses) > maxPosts: break posts = graph.get_connections(profile['id'], 'posts',**args) nargs = {} for s in posts['paging']['next'].split('&'): if 'limit=' in s: nargs['limit'] = s.split('=')[1] if 'until=' in s: nargs['until'] = s.split('=')[1] if nargs['until'] == args['until']: break else: args['limit'] = nargs['limit'] args['until'] = nargs['until'] return statuses except ConnectionError: return []
def getTweets(user,clean=True,includeRetweets=False,maxTweets=1000000,includeOtherMentions=True): twitter_access_token_key,twitter_access_token_secret,twitter_consumer_key,twitter_consumer_secret = Config.getTwitterKeys() api = TwitterAPI(twitter_access_token_key,twitter_access_token_secret, twitter_consumer_key, twitter_consumer_secret) tweets = [] li = api.fetch_by_user_names(user.twitterId,clean = clean) while li is not None and len(li) > 0: if "errors" in li and li["errors"][0]["code"] == 88: return [] for tweet in li: tweetText = str(tweet['text'].encode("utf-8")).replace('\\n', ' ').replace('\\r', '').strip()[2:-1] if not includeRetweets and tweetText.startswith('RT'): continue if not includeOtherMentions and "@" in tweet['text']: continue if clean: tweetText = re.sub(r'https?:\/\/[^\s]*','',tweetText, flags=re.MULTILINE) tweetText = re.sub(r'\\x..','',tweetText, flags=re.MULTILINE) tweetText = re.sub(r'&','and',tweetText, flags=re.IGNORECASE) tweetText = re.sub(r'[\s\t]+',' ',tweetText, flags=re.IGNORECASE) tweetText.replace('#','') tweetText = re.sub('^\s*rt\s',' ',tweetText,flags=re.IGNORECASE) if not validTweet(tweetText): continue st = Status() st.id = tweet['id'] st.likes = tweet['favorite_count'] st.shares = tweet['retweet_count'] st.CreatedOn = tweet['created_at'] st.statusType = 1 st.statusText = tweetText st.language = tweet["lang"] tweets.append(st) minId = min(tweets,key=lambda p:p.id).id li = api.fetch_by_user_names(user.twitterId,id= minId - 1,clean = clean) if len(tweets) > maxTweets or li is None or len(li) < 10: break return tweets