def get_rate_results(request, lex): results = {'success':'False'} u = request.user rating = lex[u'rating'] if(lex.has_key(u'status')): sid = int(lex[u'status']) try: status = Status.objects.get(id=sid) except Status.DoesNotExist: api = get_authorized_twython(request.session['twitter_tokens']) status_json = api.showStatus(id=lex[u'status']) status = Status.construct_from_dict(status_json) else: api = get_authorized_twython(request.session['twitter_tokens']) status_json = api.showStatus(id=lex[u'id']) status = Status.construct_from_dict(status_json) # Show user if tweet delivered from Search API, which does not have correct userid # TODO: a more elegant solution if not status.user.id: api = get_authorized_twython(request.session['twitter_tokens']) api_user = api.showUser(screen_name=status.user.screen_name) setattr(status, 'user', TwitterUserProfile.construct_from_dict(api_user)) tp = TwitterUserProfile.objects.get(id=request.session['twitter_tokens']['user_id']) prof = u.get_profile() status.save_with_user(is_cached=False) try: details = CachedStatus.objects.get(user=tp.id, status=status.id) details.prediction -= 2.0 details.save() except CachedStatus.DoesNotExist: pass if rating == 'up': rating_int = 1 elif rating == 'down': rating_int = -1 try: r = Rating.objects.get(status=status, user=tp) except: r = Rating(status=status, user=tp) # prof.whale.exp += 1 # if prof.whale.exp == prof.whale.species.evolution.minExp: prof.whale.species = prof.whale.species.evolution # prof.whale.save() r.rating = rating_int r.save() results['success'] = 'True' # results['exp'] = prof.whale.exp # results['min-exp'] = prof.whale.species.minExp # results['max-exp'] = prof.whale.species.evolution.minExp # results['species'] = prof.whale.species.img.url # results['speciesName'] = prof.whale.species.name return results
def cache_statuses(statuses, tp): s_ids = [s.id for s in statuses] s_cached = set([s.id for s in tp.cached_statuses.filter(id__in=s_ids)]) Status.savemany(statuses, is_cached=True) usercache_statuses = [] for s in statuses: if s.id in s_cached: continue s_cached.add(s.id) usercache_statuses.append(s) predictions = get_predictions(tp, usercache_statuses) CachedStatus.objects.create_in_bulk(tp, usercache_statuses, predictions)
def test_get_feed_from_other_user(self, mocked_twitter, mocked_format): api = Mock() api.GetUserTimeline.return_value = [Status(), Status()] mocked_twitter.Api.return_value = api other_user = mommy.make(settings.AUTH_USER_MODEL) other_fb_user = mommy.make( 'UserSocialAuth', user=other_user, uid='123', provider='twitter' ) feed = TwitterFeed(self.user) feed.get_feed(other_user) assert 2 == mocked_format.call_count
def mocked_post_tweet(status: str, in_reply_to_status_id: str) -> Status: if in_reply_to_status_id is None: return Status(id_str="9999_0") elif in_reply_to_status_id.startswith("0001"): return Status(id_str="9999_1") elif in_reply_to_status_id.startswith("0002"): return Status(id_str="9999_2") elif in_reply_to_status_id.startswith("0003"): return Status(id_str="9999_3") elif in_reply_to_status_id.startswith("0004"): return Status(id_str="9999_4") else: ValueError(f"Unknown in_reply_to_status_id {in_reply_to_status_id}")
def convertTruncated(tweet): raw_tweet = tweet._json if 'extended_tweet' in raw_tweet.keys(): for key, value in raw_tweet['extended_tweet'].items(): raw_tweet[key] = value converted_tweet = Status.NewFromJsonDict(raw_tweet) return converted_tweet
def update_from_stream(api, account_to_follow, include_rts=False): """Uses Twitter's streaming API to get new tweets in realtime and release them.""" normalized_account = account_to_follow.lstrip('@') # Check that account is being followed, otherwise it won't show up in our stream try: relationship = api.LookupFriendship(screen_name=normalized_account)[0] if not relationship.following: api.CreateFriendship(screen_name=normalized_account) except IndexError: api.CreateFriendship(screen_name=normalized_account) # Get timeline stream restricted to users the bot is following stream = api.GetUserStream(replies=None, withuser='******') while stream: # Make a tweet out of the stream iteration message = Status.NewFromJsonDict(stream.next()) # Check that message is a tweet, from watched account, and not RT or RTs allowed if (message.id and message.user.screen_name == normalized_account and (not message.retweeted_status or include_rts)): release_tweet(message, api) save_last_tweet(message.id)
def full_create_status(status): status = Status.create_from_dict(status) try: TwitterUserProfile.objects.get(id=status.user.id) except TwitterUserProfile.DoesNotExist: status.user.save() try: Status.objects.get(id=status.id) except: status.save() return status
def test_tweets_older_than_two_hours_are_ignored(self, mock_twitter): two_hours_ago = datetime.datetime.utcnow() - datetime.timedelta(hours=2) old_tweet_with_shift_code = Status.NewFromJsonDict({ "text": "There's a shift code here! {}".format(self.MOCK_SHIFT_CODE), "created_at": two_hours_ago.strftime(self.DATE_FMT) }) self.mock_tweets.append(old_tweet_with_shift_code) twitter_scanner = Twitter() mock_twitter.Api.return_value.GetUserTimeline.return_value = self.mock_tweets self.assertIsNone(twitter_scanner.filter())
def convert_truncated(tweet): """Converts a tweet in extended compatibility mode to a fully extended tweet. These come from the Streaming API, and python-twitter will only extract a legacy style tweet. See https://dev.twitter.com/overview/api/upcoming-changes-to-tweets for details and https://github.com/twitterdev/tweet-updates/blob/master/samples/initial/compatibilityplus_extended_13997.json for an example. This hasn't been tested extensively, so may break in some cases, but seems to work so far.""" raw_tweet = tweet._json if raw_tweet.has_key('extended_tweet'): for key, value in raw_tweet['extended_tweet'].items(): raw_tweet[key] = value converted_tweet = Status.NewFromJsonDict(raw_tweet) return converted_tweet
def ajax_search(request): if not request.user.is_authenticated() or 'twitter_tokens' not in request.session: return HttpResponse("") if request.is_ajax(): term = request.GET.get('q') page = request.GET.get('page') if (term is not None and page is not None): api = get_authorized_twython(request.session['twitter_tokens']) statuses = Status.construct_from_search_dicts(api.searchTwitter(q=term, page=page)[u'results']) return render_to_response('twitter/status_list.html', { 'statuses': statuses }, context_instance=RequestContext(request)) return HttpResponse('')
def main(): configFile = 'twitter_token.conf' locale.setlocale(locale.LC_ALL, "") tToken = tokenTwitter() tToken.setTokenFromFile(configFile) api = apiFromConfig(tToken) config = tToken.configParam(configFile) words_to_track = config.get('settings', 'words_to_track').split(',') try: for stream in api.GetStreamFilter(track=words_to_track): message = Status.NewFromJsonDict(stream) if (message.text[:2] != 'RT'): releaseTweet(message, api, config) except (NoOptionError, NoSectionError) as e: stream = None
def test_twitter_scanner_returns_link_to_tweets_that_contain_shift_codes(self, mock_twitter): tweet_with_shift_code = Status.NewFromJsonDict({ "id": 1000, "user": {"screen_name": "screenname"}, "text": "There's a shift code here! {}".format(self.MOCK_SHIFT_CODE), "created_at": self.created_at_now }) self.mock_tweets.append(tweet_with_shift_code) twitter_scanner = Twitter() mock_twitter.Api.return_value.GetUserTimeline.return_value = self.mock_tweets expected_url = 'https://twitter.com/{}/status/{}'.format( tweet_with_shift_code.user.screen_name, tweet_with_shift_code.id ) self.assertEqual(expected_url, twitter_scanner.filter())
def ingest_tweet(tweet, session, twitter_api, tweet_id_queue): """Actually ingest a single tweet, dealing with the required enqueuing.""" if not isinstance(tweet, Status): tweet = Status.NewFromJsonDict(tweet) if tweet.retweeted_status: # We don't actually care about retweets, they aren't original content. # Just insert the original. ingest_tweet(tweet.retweeted_status, session, twitter_api, tweet_id_queue) ingest_user_object(tweet.user, session) else: flag = have_tweet(session, tweet.id) t = bt.insert_tweet(session, twitter_api, tweet) if not flag: log.info(t) if tweet.in_reply_to_status_id: # This tweet is a reply. It links to some other tweet. Or possibly tweets depending on the # link content which may link many statuses. However Twitter only considers one status to # be the "reply" target. Create a "reply_to" relationship for the post we're inserting by # inserting its parent post(s) (recursively!) thread_id = str(tweet.in_reply_to_status_id) if not have_tweet(session, thread_id): tweet_id_queue.put(thread_id) pass if tweet.quoted_status: # This is a quote tweet (possibly subtweet or snarky reply, quote tweets have different # broadcast mechanics). ingest_tweet(tweet.quoted_status, session, twitter_api, tweet_id_queue) for url in tweet.urls or []: tweet_id = bt.tweet_id_from_url(url.expanded_url) if tweet_id and not have_tweet(session, tweet_id): tweet_id_queue.put(tweet_id) pass for user in tweet.user_mentions or []: if not isinstance(user, User): user = User.NewFromJsonDict(user) ingest_user_object(user, session)
def get_status(status_id: int) -> Status: cache = get_cache("status", status_id) retries = 0 if cache is None: print("Making network request for status id %s" % status_id) while True: try: data = api.GetStatus(status_id, trim_user=True) break except ConnectionError: retries += 1 if retries > 1: raise cache = write_cache("status", data) if "current_user_retweet" in cache: cache.pop("current_user_retweet") return Status.NewFromJsonDict(cache)
def get_tweets_to_rss_feed(limit: int): """Get all tweets that are known to have not been included in RSS feeds yet. Read them in order, and reverse before passing. Reading them in reverse directly will lead to reading older tweets after newer ones.""" with _get_conn() as conn: cursor = conn.cursor() if limit > 0: limit_clause = ' LIMIT {}'.format(limit) else: limit_clause = '' cursor.execute( 'SELECT tweet_json FROM {} WHERE {} = 0 ORDER BY id{}'.format( STATUS_TABLE, RSS_COLUMN, limit_clause)) tweets = [ Status.NewFromJsonDict(json.loads(row[0])) for row in cursor.fetchall() ] tweets.reverse() return tweets
def Tweet(request, status_id, account=None): output = {} status = Status.by_id(status_id) if not status: status = collector.status(status_id, account) if status: encodeURLs(status, account) output['status'] = render_to_string('status.html', {'status': status}) output['next_status'] = str(status.in_reply_to_status_id) else: output['status'] = '<li>The next tweet no longer exists</li>' output['next_status'] = str(None) if account: output['api_calls'] = account.rate_remaining response = HttpResponse() response.write(simplejson.dumps(output)) return response
def Tweet(request,status_id,account=None): output = {} status = Status.by_id(status_id) if not status: status = collector.status(status_id,account) if status: encodeURLs(status, account) output['status'] = render_to_string('status.html', {'status':status}) output['next_status'] = str(status.in_reply_to_status_id) else: output['status'] = '<li>The next tweet no longer exists</li>' output['next_status'] = str(None) if account: output['api_calls'] = account.rate_remaining response = HttpResponse() response.write(simplejson.dumps(output)) return response
def timeline(request): user = request.user if not user.is_authenticated() or 'twitter_tokens' not in request.session: return render_to_response('landing.html') twitter_tokens = request.session['twitter_tokens'] tp = TwitterUserProfile.objects.get(id=twitter_tokens['user_id']) api = get_authorized_twython(twitter_tokens) statuses = Status.construct_from_dicts(api.getFriendsTimeline(include_rts=True)) cache_timeline_backfill.delay(tp, twitter_tokens, statuses) friends = api.getFriendsStatus() Rating.appendTo(statuses, tp) return render_to_response('twitter/timeline.html', { 'statuses': statuses, 'friends': friends, 'feedtype': 'normal' }, context_instance=RequestContext(request))
def search(request): if not request.user.is_authenticated() or 'twitter_tokens' not in request.session: return HttpResponseRedirect("/") term = request.GET.get('q') if term is not None: prof = request.user.get_profile() twitter_tokens = request.session['twitter_tokens'] api = get_authorized_twython(twitter_tokens) tp = TwitterUserProfile.objects.get(id=twitter_tokens['user_id']) statuses = Status.construct_from_search_dicts(api.searchTwitter(q=term)[u'results']) friends = api.getFriendsStatus() Rating.appendTo(statuses, tp) return render_to_response('twitter/search_index.html', { # 'whale': prof.whale, 'friends': friends, 'statuses': statuses, 'term': term }, context_instance=RequestContext(request)) else: return HttpResponseRedirect("/")
def public_profile(request, username): if request.user.is_authenticated() and 'twitter_tokens' in request.session: twitter_tokens = request.session['twitter_tokens'] api = get_authorized_twython(twitter_tokens) else: # Require login return HttpResponseRedirect("/") friend = api.showUser(screen_name=username) friends = api.getFriendsStatus() prof = request.user.get_profile() tp = TwitterUserProfile.objects.get(user=prof) follow_request_sent = True is_true_friend = friend['following'] is_me = tp.id == friend['id'] if not is_true_friend: is_true_friend = False outgoing = api.friendshipsOutgoing() follow_request_sent = False if friend['id'] in outgoing['ids']: # if we have already requested to follow this person follow_request_sent = True if friend['protected'] and not is_true_friend: statuses = None else: try: statuses = Status.construct_from_dicts(api.getUserTimeline(screen_name=username)) Rating.appendTo(statuses, tp) except TwythonError: statuses = None return render_to_response('twitter/public_profile.html', { 'friends': friends, 'username': username, 'friend': friend, 'is_true_friend' : is_true_friend, 'is_me' : is_me, 'profile_protected' : friend['protected'], 'follow_request_sent': follow_request_sent, 'statuses' : statuses, }, context_instance=RequestContext(request))
def cache_timeline_backfill_callback(sender, **kwargs): """ Backfill cached timeline from the oldest tweet in statuses to the cached_time in TwitterUserProfile or 72 hours, whichever is sooner""" statuses, tp = kwargs['statuses'], kwargs['twitter_user_profile'] twitter_tokens = kwargs['twitter_tokens'] api = get_authorized_twython(twitter_tokens) oldest_time = datetime.now()-timedelta(hours=72) """ backfill_start = min(statuses, key=lambda x: x.created_at) backfill_end = max([tp.cached_time, oldest_time]) if backfill_start < backfill_end: return """ backfill_maxid = min(statuses, key=lambda x: x.id).id try: backfill_minid = max(tp.cached_statuses.filter(created_at__gt=oldest_time), key=lambda x: x.id).id if backfill_maxid < backfill_minid: return except IndexError: backfill_minid = None # print "backfill minid: " + str(backfill_minid) # print "backfill maxid: " + str(backfill_maxid) cache_timeline_signal.send(sender=sender, statuses=statuses, twitter_user_profile=tp) finished = False total_num_statuses = len(statuses) while not finished: recieved_statuses = Status.construct_from_dicts( api.getFriendsTimeline(count=200, include_rts=True, max_id=backfill_maxid, min_id=backfill_minid)) total_num_statuses += len(recieved_statuses) cache_timeline_signal.send(sender=sender, statuses=recieved_statuses, twitter_user_profile=tp) if total_num_statuses >= 600 or len(recieved_statuses) < 200: finished = True else: backfill_maxid = statuses[-1].id
def oldestStatus(self): oldest_status_id = None for member in self.members_list(): if oldest_status_id is None or oldest_status_id > member.oldest_status_id: oldest_status_id = member.oldest_status_id return Status.by_id(oldest_status_id)
def normal_timeline(api, tp, page, maxid): if tp.cached_maxid >= maxid >= tp.cached_minid: return tp.cached_statuses.filter(id__lt=maxid)[:20] else: return Status.construct_from_dicts(api.getFriendsTimeline(page=page))
def execute(self): status = Status.by_id(self.status_id)
def latestStatus(self): latest_status_id = None for member in self.members_list(): if latest_status_id is None or latest_status_id < member.latest_status_id: latest_status_id = member.latest_status_id return Status.by_id(latest_status_id)
def setUp(self): self.created_at_now = datetime.datetime.utcnow().strftime(self.DATE_FMT) self.mock_tweets = [Status.NewFromJsonDict({ "text": c, "created_at": self.created_at_now }) for i, c in enumerate(['a', 'b', 'c'])]
from twitter.models import Status, User MENTIONED_TWEETS = [ Status(id_str="0001_1", user=User(screen_name="user0001")), Status(id_str="0001_2", user=User(screen_name="user0001")), Status(id_str="0002_1", user=User(screen_name="user0002")), Status(id_str="0003_1", user=User(screen_name="user0003")), Status(id_str="0004_1", user=User(screen_name="user0004")), ] SELF_TWEETS = [ Status(id_str="9999_1"), Status(id_str="9999_2"), Status(id_str="9999_3", in_reply_to_status_id="0003_1"), Status(id_str="9999_4") ]
def cache_clean(): Status.clear_cache()
def cache_timeline_backfill(tp, twitter_tokens, statuses): """ Backfill cached timeline from the oldest tweet in statuses to the cached_time in TwitterUserProfile or 72 hours, whichever is sooner""" api = get_authorized_twython(twitter_tokens) cutoff_time = datetime.utcnow()-timedelta(hours=72) if not statuses: statuses = Status.construct_from_dicts(api.getFriendsTimeline(include_rts=True)) backfill_maxid = min(statuses, key=lambda x: x.id).id backfill_newestid = max(statuses, key=lambda x: x.id).id # Maxid and minid indicate contiguous cached status ids minid = getattr(tp, 'cached_minid', 0) maxid = getattr(tp, 'cached_maxid', 0) # No new tweets at all if backfill_newestid == maxid: return # Only one page of new tweets - just cache these if backfill_maxid < maxid: cache_statuses(statuses, tp) return # Cache as far back as 800 tweets or 72 hours worth num_apicalls = 1 finished = False total_num_statuses = len(statuses) while not finished: print "backfill minid: " + str(maxid) print "backfill maxid: " + str(backfill_maxid) recieved_statuses = Status.construct_from_dicts( api.getFriendsTimeline(count=200, include_rts=True, max_id=backfill_maxid, min_id=maxid)) num_apicalls += 1 total_num_statuses += len(recieved_statuses) statuses.extend(recieved_statuses) oldest_status = min(recieved_statuses, key=lambda x: x.id) if (oldest_status.created_at < cutoff_time or oldest_status.id <= maxid or num_apicalls >= 5): finished = True else: backfill_maxid = oldest_status.id # Set new minid, maxid for contiguous cached statuses if oldest_status.id <= maxid: tp.cached_minid = minid else: tp.cached_minid = oldest_status.id tp.cached_maxid = backfill_newestid tp.save() # If less than 50 rated statuses: force train user classifier if Rating.objects.filter(user=tp).count() < 50: force_train(tp) # print "num apicalls: " + str(num_apicalls) cache_statuses(statuses, tp)
] } profiles = [] for i in range(5): np = User() np.id = 'user%s' % i np.screen_name = 'username%s' % i for c in confs: if c == 'description': np.description = confs[c][i] elif c == 'followers_count': np.followers_count = confs[c][i] elif c == 'location': np.location = confs[c][i] elif c == 'statuses_count': np.statuses_count = confs[c][i] elif c == 'created_at': np.created_at = confs[c][i] elif c == 'profile.status.created_at': ns = Status() ns.created_at = confs[c][i] ns.text = 'text' np.status = ns profiles.append(np) print(profiles) for s in settings: for p in profiles: filterprofile(p, s)
def PostUpdate(self, status, in_reply_to_status_id=None): if len(status) > 140: raise ('Too many characters!') return Status.NewFromJsonDict({'id': 100})