def build_thread_last_to_first(api, last_twit_id):
posts_illustrator = {}
last_post = Status()
last_post.in_reply_to_status_id = last_twit_id
while last_post.in_reply_to_status_id:
last_post = api.get_status(id=last_post.in_reply_to_status_id, tweet_mode='extended')
print_status(last_post)
write_status_to_file(last_post)
posts_illustrator[last_post.id] = {
'id': last_post.id,
'text': last_post.full_text,
'in_reply_to_status_id': last_post.in_reply_to_status_id
}
def mock_mention_replies_to_another_tweet(mock_tweepy, twitter_user):
original_tweet = Status()
original_tweet.id = 1
original_tweet.full_text = "$AMZN is a great buy in 2021"
reply_tweet = Status()
reply_tweet.id = 2
reply_tweet.text = "@stock_reminder remind me of this in 1 year"
reply_tweet.user = twitter_user
reply_tweet.in_reply_to_status_id = original_tweet.id
mock_tweepy.return_value.mentions_timeline.return_value = [reply_tweet]
mock_tweepy.return_value.get_status.return_value = original_tweet
return mock_tweepy
def read_as_status(archivepath, since_id=None, max_id=None):
api = API()
since_id = since_id or 0
max_id = max_id or float("inf")
for tweet in archive.read_json(archivepath):
try:
tweet['created_at'] = fix_timeformat(tweet['created_at'])
except AttributeError:
tweet['created_at'] = fix_timeformat(tweet['timestamp'])
if tweet.get('retweeted_status'):
tweet['retweeted_status']['created_at'] = fix_timeformat(
tweet['retweeted_status']['created_at'])
try:
i = int(tweet['id_str'])
except AttributeError:
i = int(tweet['tweet_id'])
tweet['id_str'] = tweet['tweet_id']
if i > since_id and i < max_id:
yield Status.parse(api, tweet)
def test_shortreply_with_mentions(self):
"""
A reply from chromachipper uses the short reply where necessary.
"""
# This will cause chromachipper to create a reply with short reply text.
status_morethan140_char_obj = copy.deepcopy(self.status_base_obj)
status_morethan140_char_obj['text'] = u'@chromachipper #000 @abcdefghijklmno @pqrstuvwxyzabcd @efghijklmnopqur @stuvqxyzabcdefg @hijklmnopqrstuv @wxyzabcdefghijk @ashortreply'
status_morethan140_char_obj['entities']['user_mentions'] = [
{u'id': 00000000, u'indices': [0, 14], u'id_str': u'00000000', u'screen_name': u'chromachipper', u'name': u'Chroma Chipper'},
{u'id': 11111111, u'indices': [85, 96], u'id_str': u'11111111', u'screen_name': u'abcdefghijklmno', u'name': u'abcdefghijklmno'},
{u'id': 22222222, u'indices': [85, 96], u'id_str': u'22222222', u'screen_name': u'pqrstuvwxyzabcd', u'name': u'pqrstuvwxyzabcd'},
{u'id': 33333333, u'indices': [85, 96], u'id_str': u'33333333', u'screen_name': u'efghijklmnopqur', u'name': u'efghijklmnopqur'},
{u'id': 44444444, u'indices': [85, 96], u'id_str': u'44444444', u'screen_name': u'stuvqxyzabcdefg', u'name': u'stuvqxyzabcdefg'},
{u'id': 55555555, u'indices': [85, 96], u'id_str': u'55555555', u'screen_name': u'hijklmnopqrstuv', u'name': u'hijklmnopqrstuv'},
{u'id': 66666666, u'indices': [85, 96], u'id_str': u'66666666', u'screen_name': u'wxyzabcdefghijk', u'name': u'wxyzabcdefghijk'},
{u'id': 77777777, u'indices': [85, 96], u'id_str': u'77777777', u'screen_name': u'ashortreply', u'name': u'ashortreply'},
]
status_140_char = Status.parse(api=API(), json=status_morethan140_char_obj)
listener = ChromachipStreamListener(api=self.api, twitter_id=00000000)
listener.on_status(status_140_char)
self.assertEqual(self.api.last_status, u"@IWantToGift2U sent this! @abcdefghijklmno @pqrstuvwxyzabcd @efghijklmnopqur @stuvqxyzabcdefg @hijklmnopqrstuv @wxyzabcdefghijk @ashortreply")
self.assertEqual(len(self.api.last_status), 140)
def mock_mention_replies_to_extended_tweet(mock_tweepy, twitter_user):
original_tweet = Status()
original_tweet.id = 1
original_tweet.full_text = ("You wouldn’t believe it, but right now, "
"I think we’re looking at potentially 100% "
"stock returns in ~3 years for:\n\n1. $AMZN "
"at $3,055\n2. $TSLA at $661\n3. $JNJ at "
"$160\n Zero interest in debating!")
reply_tweet = Status()
reply_tweet.id = 2
reply_tweet.text = "@stock_reminder remind me of this in 3 years"
reply_tweet.user = twitter_user
reply_tweet.in_reply_to_status_id = original_tweet.id
mock_tweepy.return_value.mentions_timeline.return_value = [reply_tweet]
mock_tweepy.return_value.get_status.return_value = original_tweet
return mock_tweepy
def status(twitter_user):
tweet = Status()
tweet.id = 1
tweet.text = "Price of $AMZN in 3 months."
tweet.user = twitter_user
tweet.in_reply_to_status_id = None
return tweet
def setUp(self):
status_base_obj = {
u'contributors': None,
u'truncated': False,
u'in_reply_to_status_id': None,
u'id': 487394915643842560,
u'retweeted': False,
u'entities': {
u'symbols': [],
u'hashtags': [{u'indices': [97, 104], u'text': u'00b9f1'}],
u'urls': []},
u'in_reply_to_screen_name': u'chromachipper',
u'id_str': u'487394915643842560',
u'in_reply_to_user_id': 00000000,
u'user': {
u'id': 725633,
u'id_str': u'725633',
u'screen_name': u'IWantToGive',
u'name': u'I am a Giver',
},
}
status_with_mentions_obj = copy.deepcopy(status_base_obj)
status_with_mentions_obj['text'] = u'@chromachipper what about if I mention another account. Can I gift someone a colour? @GiftToMe #00b9f1'
status_with_mentions_obj['entities']['user_mentions'] = [
{u'id': 00000000, u'indices': [0, 14], u'id_str': u'00000000', u'screen_name': u'chromachipper', u'name': u'Chroma Chipper'},
{u'id': 11111111, u'indices': [85, 96], u'id_str': u'11111111', u'screen_name': u'GiftToMe', u'name': u'Gift Tome'},
{u'id': 22222222, u'indices': [85, 96], u'id_str': u'22222222', u'screen_name': u'GiftToMeToo', u'name': u'Gift Tometoo'}
]
status_without_mentions_obj = copy.deepcopy(status_base_obj)
status_without_mentions_obj['text'] = u'@chromachipper send me a Chroma Chip! #00b9f1'
status_without_mentions_obj['entities']['user_mentions'] = [
{u'id': 00000000, u'indices': [0, 14], u'id_str': u'00000000', u'screen_name': u'chromachipper', u'name': u'Chroma Chipper'}
]
self.api = MockAPI()
self.status_with_mentions = Status.parse(api=API(), json=status_with_mentions_obj)
self.status_without_mentions = Status.parse(api=API(), json=status_without_mentions_obj)
def on_data(self, data):
if not data:
return
json_data = json.loads(data)
api = self.__api
if not isinstance(json_data, dict):
return
if 'event' in json_data:
if json_data['event']=='follow':
target = User.parse(api, json_data["target"])
source = User.parse(api, json_data["source"])
if self.on_follow(target, source) is False:
return False
elif json_data['event']=='favorite':
target = Status.parse(api, json_data['target_object'])
source = User.parse(api, json_data['source'])
if self.on_favorite(target, source) is False:
return False
elif json_data['event']=='unfavorite':
target = Status.parse(api, json_data['target_object'])
source = User.parse(api, json_data['source'])
if self.on_unfavorite(target, source) is False:
return False
elif 'delete' in json_data:
delete = json_data['delete']['status']
if self.on_delete(delete['id'], delete['user_id']) is False:
return False
elif 'in_reply_to_status_id' in json_data:
status = Status.parse(api, json_data)
if self.on_status(status) is False:
return False
elif 'limit' in json_data:
if self.on_limit(json_data['limit']['track']) is False:
return False
def init_on_load(self):
self._words = []
self._keywords = []
self._hashtags = []
self.status = None
if self.raw_json:
self.status = Status.parse(None, json.loads(self.raw_json))
if self.status:
self.id = self.status.id
self.author_name = self.status.author.screen_name
self.create_time = self.status.created_at
self._hashtags = ['#%s' % h['text'].lower() for h in self.status.entities.get('hashtags', [])]
self._words = self.status.text.split(' ')
def on_data(self, data):
if not data:
return
json_data = json.loads(data)
api = self.__api
if not isinstance(json_data, dict):
return
if 'event' in json_data:
if json_data['event'] == 'follow':
target = User.parse(api, json_data["target"])
source = User.parse(api, json_data["source"])
if self.on_follow(target, source) is False:
return False
elif json_data['event'] == 'favorite':
target = Status.parse(api, json_data['target_object'])
source = User.parse(api, json_data['source'])
if self.on_favorite(target, source) is False:
return False
elif json_data['event'] == 'unfavorite':
target = Status.parse(api, json_data['target_object'])
source = User.parse(api, json_data['source'])
if self.on_unfavorite(target, source) is False:
return False
elif 'delete' in json_data:
delete = json_data['delete']['status']
if self.on_delete(delete['id'], delete['user_id']) is False:
return False
elif 'in_reply_to_status_id' in json_data:
status = Status.parse(api, json_data)
if self.on_status(status) is False:
return False
elif 'limit' in json_data:
if self.on_limit(json_data['limit']['track']) is False:
return False
def on_data(self, data):
"""Called when raw data is received from connection.
Override this method if you wish to manually handle
the stream data. Return False to stop stream and close connection.
"""
if 'in_reply_to_status_id' in data and '"lang":"en"' in data:
status = Status.parse(self.api, json.loads(data))
if self.on_status(status) is False:
return False
elif 'delete' in data:
delete = json.loads(data)['delete']['status']
if self.on_delete(delete['id'], delete['user_id']) is False:
return False
elif 'limit' in data:
if self.on_limit(json.loads(data)['limit']['track']) is False:
return False
def on_data(self, data):
"""Called when raw data is received from connection.
Override this method if you wish to manually handle
the stream data. Return False to stop stream and close connection.
"""
if 'in_reply_to_status_id' in data and '"lang":"en"' in data:
status = Status.parse(self.api, json.loads(data))
if self.on_status(status) is False:
return False
elif 'delete' in data:
delete = json.loads(data)['delete']['status']
if self.on_delete(delete['id'], delete['user_id']) is False:
return False
elif 'limit' in data:
if self.on_limit(json.loads(data)['limit']['track']) is False:
return False
def process_twitter_data(
status: Status,
twitter_credentials: dict,
add_sentiment: bool = True,
add_bot_analysis: bool = True) -> TwitterDataOutput:
output: TwitterDataOutput = TwitterDataOutput(status={}, user={})
try:
# 1. Retrieve status document
status_doc: StatusDoc = TwitterDataProcessor.process_status(
status=status)
response_sent: SentimentAnalysisOutput = SentimentAnalysisOutput()
# 2. Add additional parameters related to sentiment analysis
if add_sentiment:
if len(status_doc.text) >= TwitterDataProcessor.min_char:
response_sent: SentimentAnalysisOutput = TwitterDataProcessor.process_sentiment_analysis(
doc=status_doc.text)
status_doc.sentiment_analysis: dict = response_sent.__dict__
# 3. Get the user
user: User = status.__getattribute__("user")
user_doc: UserAccountDoc = TwitterDataProcessor.process_user(
user=user)
# 4. Add additional params
if add_bot_analysis:
response_botometer_analysis: BotometerAnalysisOutput = TwitterDataProcessor.process_botometer_analysis(
user_id=user_doc.id,
twitter_credentials=twitter_credentials)
user_doc.botometer_analysis: dict = response_botometer_analysis.__dict__
# 5. Get the output
output: TwitterDataOutput = TwitterDataOutput(
status=status_doc.__dict__, user=user_doc.__dict__)
except Exception as e:
logger.error(e)
return output
def on_status(self, status: Status):
logger.info(f"1. Loading Status with ID {status.__getattribute__('id')}")
# 1. Check whether the status is already in the storage
non_exists: bool = self.check_data_in_storage(
entity_id=status.__getattribute__('id'),
storage=self.storage,
collection_name=self.collection_names.get("status"),
identifier_key="id")
if non_exists:
# 3. Process Tweets an Users
logger.info(f"2. Pre-processing Status with ID {status.__getattribute__('id')}")
data: TwitterDataOutput = self.process_status(
status=status, add_sentiment=self.add_sentiment,
add_bot_analysis=self.add_bot_analysis)
logger.info(f"3. Storing Status with ID {status.__getattribute__('id')} in {self.storage.title()}")
# 2. Storage data
self.storage_data(data=data, collection_names=self.collection_names,
storage=self.storage, mongodb_connector=self.mongodb_connector,
elasticsearch_connector=self.elasticsearch_connector,
identifier_key=self.identifier_key)
def read_as_status(archivepath, since_id=None, max_id=None):
api = API()
since_id = since_id or 0
max_id = max_id or float("inf")
for tweet in archive.read_json(archivepath):
try:
tweet['created_at'] = fix_timeformat(tweet['created_at'])
except AttributeError:
tweet['created_at'] = fix_timeformat(tweet['timestamp'])
if tweet.get('retweeted_status'):
tweet['retweeted_status']['created_at'] = fix_timeformat(tweet['retweeted_status']['created_at'])
try:
i = int(tweet['id_str'])
except AttributeError:
i = int(tweet['tweet_id'])
tweet['id_str'] = tweet['tweet_id']
if i > since_id and i < max_id:
yield Status.parse(api, tweet)
def status(twitter_user):
tweet = Status()
tweet.id = 1
tweet.text = "Price of $AMZN in 3 months."
tweet.user = twitter_user
return tweet
def test_is_not_older_than_2_weeks(self):
tweet = Status()
tweet.created_at = datetime.now()
self.assertFalse(is_older_than_2_weeks(tweet))
def post_tweet(self, text: str) -> Status:
log(f"Fake Tweet Sent: \"{text}\"")
status = Status()
status._json = {"text": f"{text}"}
return status
def get_user_timeline(self, id: str) -> List[Status]:
status_1 = Status()
status_1._json = {"text": "tweet_1"}
status_2 = Status()
status_2._json = {"text": "tweet_2"}
return [status_2, status_1]
def assign_full_text_to_text(t: tweepy.Status):
"""Copy full_text attribute to text."""
t.text = t.full_text
t._json['text'] = t._json['full_text']
return t
def delete(tweet: Status):
tweet.destroy()
def status_from_dict(d: dict) -> Status:
"""Turn a dict into a Status."""
return Status().parse(None, d)
def test_is_older_than_2_weeks(self):
tweet = Status()
tweet.created_at = datetime(day=6, month=10, year=1975)
self.assertTrue(is_older_than_2_weeks(tweet))
def __create_fake_user_timeline() -> List[Status]:
status_1 = Status()
status_1._json = {"text": "tweet_1 1. #"}
status_2 = Status()
status_2._json = {"text": "tweet_2 2. #"}
return [status_2, status_1]
def retweet(tweet: tweepy.Status) -> None:
"""Retweet a tweet."""
try:
tweet.retweet()
except tweepy.TweepError as e:
utils.log_tweepy_error(e)
def status(twitter_user):
tweet = Status()
tweet.id = 1
tweet.text = "What is the current price of $BABA?"
tweet.user = twitter_user
return tweet
def like(tweet: tweepy.Status) -> None:
"""Like a tweet."""
try:
tweet.favorite()
except tweepy.TweepError as e:
utils.log_tweepy_error(e)
def status_with_multiple_stocks(twitter_user):
tweet = Status()
tweet.id = 1
tweet.text = "Remind me of $AMZN, $MSFT, $AAPL and $BABA in 3 months."
tweet.user = twitter_user
return tweet
def assign_full_text_to_text(t: tweepy.Status):
"""Copy full_text attribute to text."""
t.text = t.full_text
t._json['text'] = t._json['full_text']
return t
def get_status(text, created_at=None):
_json = {"text": text}
if created_at:
_json['created_at'] = created_at.strftime("%a %b %d %H:%M:%S %Y")
return Status().parse(object, _json)
