def convertTruncated(tweet):
raw_tweet = tweet._json
if 'extended_tweet' in raw_tweet.keys():
for key, value in raw_tweet['extended_tweet'].items():
raw_tweet[key] = value
converted_tweet = Status.NewFromJsonDict(raw_tweet)
return converted_tweet
def update_from_stream(api, account_to_follow, include_rts=False):
"""Uses Twitter's streaming API to get new tweets in realtime and release them."""
normalized_account = account_to_follow.lstrip('@')
# Check that account is being followed, otherwise it won't show up in our stream
try:
relationship = api.LookupFriendship(screen_name=normalized_account)[0]
if not relationship.following:
api.CreateFriendship(screen_name=normalized_account)
except IndexError:
api.CreateFriendship(screen_name=normalized_account)
# Get timeline stream restricted to users the bot is following
stream = api.GetUserStream(replies=None, withuser='******')
while stream:
# Make a tweet out of the stream iteration
message = Status.NewFromJsonDict(stream.next())
# Check that message is a tweet, from watched account, and not RT or RTs allowed
if (message.id and
message.user.screen_name == normalized_account and
(not message.retweeted_status or include_rts)):
release_tweet(message, api)
save_last_tweet(message.id)
def test_tweets_older_than_two_hours_are_ignored(self, mock_twitter):
two_hours_ago = datetime.datetime.utcnow() - datetime.timedelta(hours=2)
old_tweet_with_shift_code = Status.NewFromJsonDict({
"text": "There's a shift code here! {}".format(self.MOCK_SHIFT_CODE),
"created_at": two_hours_ago.strftime(self.DATE_FMT)
})
self.mock_tweets.append(old_tweet_with_shift_code)
twitter_scanner = Twitter()
mock_twitter.Api.return_value.GetUserTimeline.return_value = self.mock_tweets
self.assertIsNone(twitter_scanner.filter())
def convert_truncated(tweet):
"""Converts a tweet in extended compatibility mode to a fully extended tweet.
These come from the Streaming API, and python-twitter will only extract a legacy style tweet.
See https://dev.twitter.com/overview/api/upcoming-changes-to-tweets for details and https://github.com/twitterdev/tweet-updates/blob/master/samples/initial/compatibilityplus_extended_13997.json for an example.
This hasn't been tested extensively, so may break in some cases, but seems to work so far."""
raw_tweet = tweet._json
if raw_tweet.has_key('extended_tweet'):
for key, value in raw_tweet['extended_tweet'].items():
raw_tweet[key] = value
converted_tweet = Status.NewFromJsonDict(raw_tweet)
return converted_tweet
def main():
configFile = 'twitter_token.conf'
locale.setlocale(locale.LC_ALL, "")
tToken = tokenTwitter()
tToken.setTokenFromFile(configFile)
api = apiFromConfig(tToken)
config = tToken.configParam(configFile)
words_to_track = config.get('settings', 'words_to_track').split(',')
try:
for stream in api.GetStreamFilter(track=words_to_track):
message = Status.NewFromJsonDict(stream)
if (message.text[:2] != 'RT'):
releaseTweet(message, api, config)
except (NoOptionError, NoSectionError) as e:
stream = None
def test_twitter_scanner_returns_link_to_tweets_that_contain_shift_codes(self, mock_twitter):
tweet_with_shift_code = Status.NewFromJsonDict({
"id": 1000,
"user": {"screen_name": "screenname"},
"text": "There's a shift code here! {}".format(self.MOCK_SHIFT_CODE),
"created_at": self.created_at_now
})
self.mock_tweets.append(tweet_with_shift_code)
twitter_scanner = Twitter()
mock_twitter.Api.return_value.GetUserTimeline.return_value = self.mock_tweets
expected_url = 'https://twitter.com/{}/status/{}'.format(
tweet_with_shift_code.user.screen_name,
tweet_with_shift_code.id
)
self.assertEqual(expected_url, twitter_scanner.filter())
def get_status(status_id: int) -> Status:
cache = get_cache("status", status_id)
retries = 0
if cache is None:
print("Making network request for status id %s" % status_id)
while True:
try:
data = api.GetStatus(status_id, trim_user=True)
break
except ConnectionError:
retries += 1
if retries > 1:
raise
cache = write_cache("status", data)
if "current_user_retweet" in cache:
cache.pop("current_user_retweet")
return Status.NewFromJsonDict(cache)
def ingest_tweet(tweet, session, twitter_api, tweet_id_queue):
"""Actually ingest a single tweet, dealing with the required enqueuing."""
if not isinstance(tweet, Status):
tweet = Status.NewFromJsonDict(tweet)
if tweet.retweeted_status:
# We don't actually care about retweets, they aren't original content.
# Just insert the original.
ingest_tweet(tweet.retweeted_status, session, twitter_api,
tweet_id_queue)
ingest_user_object(tweet.user, session)
else:
flag = have_tweet(session, tweet.id)
t = bt.insert_tweet(session, twitter_api, tweet)
if not flag:
log.info(t)
if tweet.in_reply_to_status_id:
# This tweet is a reply. It links to some other tweet. Or possibly tweets depending on the
# link content which may link many statuses. However Twitter only considers one status to
# be the "reply" target. Create a "reply_to" relationship for the post we're inserting by
# inserting its parent post(s) (recursively!)
thread_id = str(tweet.in_reply_to_status_id)
if not have_tweet(session, thread_id):
tweet_id_queue.put(thread_id)
pass
if tweet.quoted_status:
# This is a quote tweet (possibly subtweet or snarky reply, quote tweets have different
# broadcast mechanics).
ingest_tweet(tweet.quoted_status, session, twitter_api,
tweet_id_queue)
for url in tweet.urls or []:
tweet_id = bt.tweet_id_from_url(url.expanded_url)
if tweet_id and not have_tweet(session, tweet_id):
tweet_id_queue.put(tweet_id)
pass
for user in tweet.user_mentions or []:
if not isinstance(user, User):
user = User.NewFromJsonDict(user)
ingest_user_object(user, session)
def get_tweets_to_rss_feed(limit: int):
"""Get all tweets that are known to have not been included in RSS feeds yet. Read them in order, and reverse
before passing. Reading them in reverse directly will lead to reading older tweets after newer ones."""
with _get_conn() as conn:
cursor = conn.cursor()
if limit > 0:
limit_clause = ' LIMIT {}'.format(limit)
else:
limit_clause = ''
cursor.execute(
'SELECT tweet_json FROM {} WHERE {} = 0 ORDER BY id{}'.format(
STATUS_TABLE, RSS_COLUMN, limit_clause))
tweets = [
Status.NewFromJsonDict(json.loads(row[0]))
for row in cursor.fetchall()
]
tweets.reverse()
return tweets
def setUp(self):
self.created_at_now = datetime.datetime.utcnow().strftime(self.DATE_FMT)
self.mock_tweets = [Status.NewFromJsonDict({
"text": c,
"created_at": self.created_at_now
}) for i, c in enumerate(['a', 'b', 'c'])]
def PostUpdate(self, status, in_reply_to_status_id=None):
if len(status) > 140:
raise ('Too many characters!')
return Status.NewFromJsonDict({'id': 100})
