def get_rate_results(request, lex):
results = {'success':'False'}
u = request.user
rating = lex[u'rating']
if(lex.has_key(u'status')):
sid = int(lex[u'status'])
try: status = Status.objects.get(id=sid)
except Status.DoesNotExist:
api = get_authorized_twython(request.session['twitter_tokens'])
status_json = api.showStatus(id=lex[u'status'])
status = Status.construct_from_dict(status_json)
else:
api = get_authorized_twython(request.session['twitter_tokens'])
status_json = api.showStatus(id=lex[u'id'])
status = Status.construct_from_dict(status_json)
# Show user if tweet delivered from Search API, which does not have correct userid
# TODO: a more elegant solution
if not status.user.id:
api = get_authorized_twython(request.session['twitter_tokens'])
api_user = api.showUser(screen_name=status.user.screen_name)
setattr(status, 'user', TwitterUserProfile.construct_from_dict(api_user))
tp = TwitterUserProfile.objects.get(id=request.session['twitter_tokens']['user_id'])
prof = u.get_profile()
status.save_with_user(is_cached=False)
try:
details = CachedStatus.objects.get(user=tp.id, status=status.id)
details.prediction -= 2.0
details.save()
except CachedStatus.DoesNotExist: pass
if rating == 'up':
rating_int = 1
elif rating == 'down':
rating_int = -1
try:
r = Rating.objects.get(status=status, user=tp)
except:
r = Rating(status=status, user=tp)
# prof.whale.exp += 1
# if prof.whale.exp == prof.whale.species.evolution.minExp: prof.whale.species = prof.whale.species.evolution
# prof.whale.save()
r.rating = rating_int
r.save()
results['success'] = 'True'
# results['exp'] = prof.whale.exp
# results['min-exp'] = prof.whale.species.minExp
# results['max-exp'] = prof.whale.species.evolution.minExp
# results['species'] = prof.whale.species.img.url
# results['speciesName'] = prof.whale.species.name
return results
def cache_statuses(statuses, tp):
s_ids = [s.id for s in statuses]
s_cached = set([s.id for s in tp.cached_statuses.filter(id__in=s_ids)])
Status.savemany(statuses, is_cached=True)
usercache_statuses = []
for s in statuses:
if s.id in s_cached: continue
s_cached.add(s.id)
usercache_statuses.append(s)
predictions = get_predictions(tp, usercache_statuses)
CachedStatus.objects.create_in_bulk(tp, usercache_statuses, predictions)
def test_get_feed_from_other_user(self, mocked_twitter, mocked_format):
api = Mock()
api.GetUserTimeline.return_value = [Status(), Status()]
mocked_twitter.Api.return_value = api
other_user = mommy.make(settings.AUTH_USER_MODEL)
other_fb_user = mommy.make(
'UserSocialAuth', user=other_user, uid='123', provider='twitter'
)
feed = TwitterFeed(self.user)
feed.get_feed(other_user)
assert 2 == mocked_format.call_count
def mocked_post_tweet(status: str, in_reply_to_status_id: str) -> Status:
if in_reply_to_status_id is None:
return Status(id_str="9999_0")
elif in_reply_to_status_id.startswith("0001"):
return Status(id_str="9999_1")
elif in_reply_to_status_id.startswith("0002"):
return Status(id_str="9999_2")
elif in_reply_to_status_id.startswith("0003"):
return Status(id_str="9999_3")
elif in_reply_to_status_id.startswith("0004"):
return Status(id_str="9999_4")
else:
ValueError(f"Unknown in_reply_to_status_id {in_reply_to_status_id}")
def convertTruncated(tweet):
raw_tweet = tweet._json
if 'extended_tweet' in raw_tweet.keys():
for key, value in raw_tweet['extended_tweet'].items():
raw_tweet[key] = value
converted_tweet = Status.NewFromJsonDict(raw_tweet)
return converted_tweet
def update_from_stream(api, account_to_follow, include_rts=False):
"""Uses Twitter's streaming API to get new tweets in realtime and release them."""
normalized_account = account_to_follow.lstrip('@')
# Check that account is being followed, otherwise it won't show up in our stream
try:
relationship = api.LookupFriendship(screen_name=normalized_account)[0]
if not relationship.following:
api.CreateFriendship(screen_name=normalized_account)
except IndexError:
api.CreateFriendship(screen_name=normalized_account)
# Get timeline stream restricted to users the bot is following
stream = api.GetUserStream(replies=None, withuser='******')
while stream:
# Make a tweet out of the stream iteration
message = Status.NewFromJsonDict(stream.next())
# Check that message is a tweet, from watched account, and not RT or RTs allowed
if (message.id and
message.user.screen_name == normalized_account and
(not message.retweeted_status or include_rts)):
release_tweet(message, api)
save_last_tweet(message.id)
def full_create_status(status):
status = Status.create_from_dict(status)
try: TwitterUserProfile.objects.get(id=status.user.id)
except TwitterUserProfile.DoesNotExist:
status.user.save()
try: Status.objects.get(id=status.id)
except: status.save()
return status
def test_tweets_older_than_two_hours_are_ignored(self, mock_twitter):
two_hours_ago = datetime.datetime.utcnow() - datetime.timedelta(hours=2)
old_tweet_with_shift_code = Status.NewFromJsonDict({
"text": "There's a shift code here! {}".format(self.MOCK_SHIFT_CODE),
"created_at": two_hours_ago.strftime(self.DATE_FMT)
})
self.mock_tweets.append(old_tweet_with_shift_code)
twitter_scanner = Twitter()
mock_twitter.Api.return_value.GetUserTimeline.return_value = self.mock_tweets
self.assertIsNone(twitter_scanner.filter())
def convert_truncated(tweet):
"""Converts a tweet in extended compatibility mode to a fully extended tweet.
These come from the Streaming API, and python-twitter will only extract a legacy style tweet.
See https://dev.twitter.com/overview/api/upcoming-changes-to-tweets for details and https://github.com/twitterdev/tweet-updates/blob/master/samples/initial/compatibilityplus_extended_13997.json for an example.
This hasn't been tested extensively, so may break in some cases, but seems to work so far."""
raw_tweet = tweet._json
if raw_tweet.has_key('extended_tweet'):
for key, value in raw_tweet['extended_tweet'].items():
raw_tweet[key] = value
converted_tweet = Status.NewFromJsonDict(raw_tweet)
return converted_tweet
def ajax_search(request):
if not request.user.is_authenticated() or 'twitter_tokens' not in request.session:
return HttpResponse("")
if request.is_ajax():
term = request.GET.get('q')
page = request.GET.get('page')
if (term is not None and page is not None):
api = get_authorized_twython(request.session['twitter_tokens'])
statuses = Status.construct_from_search_dicts(api.searchTwitter(q=term, page=page)[u'results'])
return render_to_response('twitter/status_list.html', {
'statuses': statuses
}, context_instance=RequestContext(request))
return HttpResponse('')
def main():
configFile = 'twitter_token.conf'
locale.setlocale(locale.LC_ALL, "")
tToken = tokenTwitter()
tToken.setTokenFromFile(configFile)
api = apiFromConfig(tToken)
config = tToken.configParam(configFile)
words_to_track = config.get('settings', 'words_to_track').split(',')
try:
for stream in api.GetStreamFilter(track=words_to_track):
message = Status.NewFromJsonDict(stream)
if (message.text[:2] != 'RT'):
releaseTweet(message, api, config)
except (NoOptionError, NoSectionError) as e:
stream = None
def test_twitter_scanner_returns_link_to_tweets_that_contain_shift_codes(self, mock_twitter):
tweet_with_shift_code = Status.NewFromJsonDict({
"id": 1000,
"user": {"screen_name": "screenname"},
"text": "There's a shift code here! {}".format(self.MOCK_SHIFT_CODE),
"created_at": self.created_at_now
})
self.mock_tweets.append(tweet_with_shift_code)
twitter_scanner = Twitter()
mock_twitter.Api.return_value.GetUserTimeline.return_value = self.mock_tweets
expected_url = 'https://twitter.com/{}/status/{}'.format(
tweet_with_shift_code.user.screen_name,
tweet_with_shift_code.id
)
self.assertEqual(expected_url, twitter_scanner.filter())
def ingest_tweet(tweet, session, twitter_api, tweet_id_queue):
"""Actually ingest a single tweet, dealing with the required enqueuing."""
if not isinstance(tweet, Status):
tweet = Status.NewFromJsonDict(tweet)
if tweet.retweeted_status:
# We don't actually care about retweets, they aren't original content.
# Just insert the original.
ingest_tweet(tweet.retweeted_status, session, twitter_api,
tweet_id_queue)
ingest_user_object(tweet.user, session)
else:
flag = have_tweet(session, tweet.id)
t = bt.insert_tweet(session, twitter_api, tweet)
if not flag:
log.info(t)
if tweet.in_reply_to_status_id:
# This tweet is a reply. It links to some other tweet. Or possibly tweets depending on the
# link content which may link many statuses. However Twitter only considers one status to
# be the "reply" target. Create a "reply_to" relationship for the post we're inserting by
# inserting its parent post(s) (recursively!)
thread_id = str(tweet.in_reply_to_status_id)
if not have_tweet(session, thread_id):
tweet_id_queue.put(thread_id)
pass
if tweet.quoted_status:
# This is a quote tweet (possibly subtweet or snarky reply, quote tweets have different
# broadcast mechanics).
ingest_tweet(tweet.quoted_status, session, twitter_api,
tweet_id_queue)
for url in tweet.urls or []:
tweet_id = bt.tweet_id_from_url(url.expanded_url)
if tweet_id and not have_tweet(session, tweet_id):
tweet_id_queue.put(tweet_id)
pass
for user in tweet.user_mentions or []:
if not isinstance(user, User):
user = User.NewFromJsonDict(user)
ingest_user_object(user, session)
def get_status(status_id: int) -> Status:
cache = get_cache("status", status_id)
retries = 0
if cache is None:
print("Making network request for status id %s" % status_id)
while True:
try:
data = api.GetStatus(status_id, trim_user=True)
break
except ConnectionError:
retries += 1
if retries > 1:
raise
cache = write_cache("status", data)
if "current_user_retweet" in cache:
cache.pop("current_user_retweet")
return Status.NewFromJsonDict(cache)
def get_tweets_to_rss_feed(limit: int):
"""Get all tweets that are known to have not been included in RSS feeds yet. Read them in order, and reverse
before passing. Reading them in reverse directly will lead to reading older tweets after newer ones."""
with _get_conn() as conn:
cursor = conn.cursor()
if limit > 0:
limit_clause = ' LIMIT {}'.format(limit)
else:
limit_clause = ''
cursor.execute(
'SELECT tweet_json FROM {} WHERE {} = 0 ORDER BY id{}'.format(
STATUS_TABLE, RSS_COLUMN, limit_clause))
tweets = [
Status.NewFromJsonDict(json.loads(row[0]))
for row in cursor.fetchall()
]
tweets.reverse()
return tweets
def Tweet(request, status_id, account=None):
output = {}
status = Status.by_id(status_id)
if not status:
status = collector.status(status_id, account)
if status:
encodeURLs(status, account)
output['status'] = render_to_string('status.html', {'status': status})
output['next_status'] = str(status.in_reply_to_status_id)
else:
output['status'] = '<li>The next tweet no longer exists</li>'
output['next_status'] = str(None)
if account:
output['api_calls'] = account.rate_remaining
response = HttpResponse()
response.write(simplejson.dumps(output))
return response
def Tweet(request,status_id,account=None):
output = {}
status = Status.by_id(status_id)
if not status:
status = collector.status(status_id,account)
if status:
encodeURLs(status, account)
output['status'] = render_to_string('status.html', {'status':status})
output['next_status'] = str(status.in_reply_to_status_id)
else:
output['status'] = '<li>The next tweet no longer exists</li>'
output['next_status'] = str(None)
if account:
output['api_calls'] = account.rate_remaining
response = HttpResponse()
response.write(simplejson.dumps(output))
return response
def timeline(request):
user = request.user
if not user.is_authenticated() or 'twitter_tokens' not in request.session:
return render_to_response('landing.html')
twitter_tokens = request.session['twitter_tokens']
tp = TwitterUserProfile.objects.get(id=twitter_tokens['user_id'])
api = get_authorized_twython(twitter_tokens)
statuses = Status.construct_from_dicts(api.getFriendsTimeline(include_rts=True))
cache_timeline_backfill.delay(tp, twitter_tokens, statuses)
friends = api.getFriendsStatus()
Rating.appendTo(statuses, tp)
return render_to_response('twitter/timeline.html',
{
'statuses': statuses,
'friends': friends,
'feedtype': 'normal'
},
context_instance=RequestContext(request))
def search(request):
if not request.user.is_authenticated() or 'twitter_tokens' not in request.session:
return HttpResponseRedirect("/")
term = request.GET.get('q')
if term is not None:
prof = request.user.get_profile()
twitter_tokens = request.session['twitter_tokens']
api = get_authorized_twython(twitter_tokens)
tp = TwitterUserProfile.objects.get(id=twitter_tokens['user_id'])
statuses = Status.construct_from_search_dicts(api.searchTwitter(q=term)[u'results'])
friends = api.getFriendsStatus()
Rating.appendTo(statuses, tp)
return render_to_response('twitter/search_index.html', {
# 'whale': prof.whale,
'friends': friends,
'statuses': statuses,
'term': term
}, context_instance=RequestContext(request))
else:
return HttpResponseRedirect("/")
def public_profile(request, username):
if request.user.is_authenticated() and 'twitter_tokens' in request.session:
twitter_tokens = request.session['twitter_tokens']
api = get_authorized_twython(twitter_tokens)
else: # Require login
return HttpResponseRedirect("/")
friend = api.showUser(screen_name=username)
friends = api.getFriendsStatus()
prof = request.user.get_profile()
tp = TwitterUserProfile.objects.get(user=prof)
follow_request_sent = True
is_true_friend = friend['following']
is_me = tp.id == friend['id']
if not is_true_friend:
is_true_friend = False
outgoing = api.friendshipsOutgoing()
follow_request_sent = False
if friend['id'] in outgoing['ids']: # if we have already requested to follow this person
follow_request_sent = True
if friend['protected'] and not is_true_friend:
statuses = None
else:
try:
statuses = Status.construct_from_dicts(api.getUserTimeline(screen_name=username))
Rating.appendTo(statuses, tp)
except TwythonError:
statuses = None
return render_to_response('twitter/public_profile.html',
{
'friends': friends,
'username': username,
'friend': friend,
'is_true_friend' : is_true_friend,
'is_me' : is_me,
'profile_protected' : friend['protected'],
'follow_request_sent': follow_request_sent,
'statuses' : statuses,
},
context_instance=RequestContext(request))
def cache_timeline_backfill_callback(sender, **kwargs):
""" Backfill cached timeline from the oldest tweet in statuses to
the cached_time in TwitterUserProfile or 72 hours, whichever is sooner"""
statuses, tp = kwargs['statuses'], kwargs['twitter_user_profile']
twitter_tokens = kwargs['twitter_tokens']
api = get_authorized_twython(twitter_tokens)
oldest_time = datetime.now()-timedelta(hours=72)
"""
backfill_start = min(statuses, key=lambda x: x.created_at)
backfill_end = max([tp.cached_time, oldest_time])
if backfill_start < backfill_end: return
"""
backfill_maxid = min(statuses, key=lambda x: x.id).id
try:
backfill_minid = max(tp.cached_statuses.filter(created_at__gt=oldest_time), key=lambda x: x.id).id
if backfill_maxid < backfill_minid: return
except IndexError:
backfill_minid = None
# print "backfill minid: " + str(backfill_minid)
# print "backfill maxid: " + str(backfill_maxid)
cache_timeline_signal.send(sender=sender, statuses=statuses,
twitter_user_profile=tp)
finished = False
total_num_statuses = len(statuses)
while not finished:
recieved_statuses = Status.construct_from_dicts(
api.getFriendsTimeline(count=200, include_rts=True,
max_id=backfill_maxid, min_id=backfill_minid))
total_num_statuses += len(recieved_statuses)
cache_timeline_signal.send(sender=sender, statuses=recieved_statuses,
twitter_user_profile=tp)
if total_num_statuses >= 600 or len(recieved_statuses) < 200: finished = True
else: backfill_maxid = statuses[-1].id
def oldestStatus(self):
oldest_status_id = None
for member in self.members_list():
if oldest_status_id is None or oldest_status_id > member.oldest_status_id:
oldest_status_id = member.oldest_status_id
return Status.by_id(oldest_status_id)
def oldestStatus(self):
oldest_status_id = None
for member in self.members_list():
if oldest_status_id is None or oldest_status_id > member.oldest_status_id:
oldest_status_id = member.oldest_status_id
return Status.by_id(oldest_status_id)
def normal_timeline(api, tp, page, maxid):
if tp.cached_maxid >= maxid >= tp.cached_minid:
return tp.cached_statuses.filter(id__lt=maxid)[:20]
else:
return Status.construct_from_dicts(api.getFriendsTimeline(page=page))
def execute(self):
status = Status.by_id(self.status_id)
def latestStatus(self):
latest_status_id = None
for member in self.members_list():
if latest_status_id is None or latest_status_id < member.latest_status_id:
latest_status_id = member.latest_status_id
return Status.by_id(latest_status_id)
def setUp(self):
self.created_at_now = datetime.datetime.utcnow().strftime(self.DATE_FMT)
self.mock_tweets = [Status.NewFromJsonDict({
"text": c,
"created_at": self.created_at_now
}) for i, c in enumerate(['a', 'b', 'c'])]
from twitter.models import Status, User
MENTIONED_TWEETS = [
Status(id_str="0001_1", user=User(screen_name="user0001")),
Status(id_str="0001_2", user=User(screen_name="user0001")),
Status(id_str="0002_1", user=User(screen_name="user0002")),
Status(id_str="0003_1", user=User(screen_name="user0003")),
Status(id_str="0004_1", user=User(screen_name="user0004")),
]
SELF_TWEETS = [
Status(id_str="9999_1"),
Status(id_str="9999_2"),
Status(id_str="9999_3", in_reply_to_status_id="0003_1"),
Status(id_str="9999_4")
]
def latestStatus(self):
latest_status_id = None
for member in self.members_list():
if latest_status_id is None or latest_status_id < member.latest_status_id:
latest_status_id = member.latest_status_id
return Status.by_id(latest_status_id)
def cache_clean():
Status.clear_cache()
def cache_timeline_backfill(tp, twitter_tokens, statuses):
""" Backfill cached timeline from the oldest tweet in statuses to
the cached_time in TwitterUserProfile or 72 hours, whichever is sooner"""
api = get_authorized_twython(twitter_tokens)
cutoff_time = datetime.utcnow()-timedelta(hours=72)
if not statuses:
statuses = Status.construct_from_dicts(api.getFriendsTimeline(include_rts=True))
backfill_maxid = min(statuses, key=lambda x: x.id).id
backfill_newestid = max(statuses, key=lambda x: x.id).id
# Maxid and minid indicate contiguous cached status ids
minid = getattr(tp, 'cached_minid', 0)
maxid = getattr(tp, 'cached_maxid', 0)
# No new tweets at all
if backfill_newestid == maxid: return
# Only one page of new tweets - just cache these
if backfill_maxid < maxid:
cache_statuses(statuses, tp)
return
# Cache as far back as 800 tweets or 72 hours worth
num_apicalls = 1
finished = False
total_num_statuses = len(statuses)
while not finished:
print "backfill minid: " + str(maxid)
print "backfill maxid: " + str(backfill_maxid)
recieved_statuses = Status.construct_from_dicts(
api.getFriendsTimeline(count=200, include_rts=True,
max_id=backfill_maxid, min_id=maxid))
num_apicalls += 1
total_num_statuses += len(recieved_statuses)
statuses.extend(recieved_statuses)
oldest_status = min(recieved_statuses, key=lambda x: x.id)
if (oldest_status.created_at < cutoff_time or
oldest_status.id <= maxid or
num_apicalls >= 5): finished = True
else: backfill_maxid = oldest_status.id
# Set new minid, maxid for contiguous cached statuses
if oldest_status.id <= maxid:
tp.cached_minid = minid
else:
tp.cached_minid = oldest_status.id
tp.cached_maxid = backfill_newestid
tp.save()
# If less than 50 rated statuses: force train user classifier
if Rating.objects.filter(user=tp).count() < 50:
force_train(tp)
# print "num apicalls: " + str(num_apicalls)
cache_statuses(statuses, tp)
def execute(self):
status = Status.by_id(self.status_id)
]
}
profiles = []
for i in range(5):
np = User()
np.id = 'user%s' % i
np.screen_name = 'username%s' % i
for c in confs:
if c == 'description':
np.description = confs[c][i]
elif c == 'followers_count':
np.followers_count = confs[c][i]
elif c == 'location':
np.location = confs[c][i]
elif c == 'statuses_count':
np.statuses_count = confs[c][i]
elif c == 'created_at':
np.created_at = confs[c][i]
elif c == 'profile.status.created_at':
ns = Status()
ns.created_at = confs[c][i]
ns.text = 'text'
np.status = ns
profiles.append(np)
print(profiles)
for s in settings:
for p in profiles:
filterprofile(p, s)
def PostUpdate(self, status, in_reply_to_status_id=None):
if len(status) > 140:
raise ('Too many characters!')
return Status.NewFromJsonDict({'id': 100})
