Beispiel #1
0
def build_thread_last_to_first(api, last_twit_id):
    posts_illustrator = {}
    last_post = Status()
    last_post.in_reply_to_status_id = last_twit_id
    while last_post.in_reply_to_status_id:
        last_post = api.get_status(id=last_post.in_reply_to_status_id, tweet_mode='extended')
        print_status(last_post)
        write_status_to_file(last_post)
        posts_illustrator[last_post.id] = {
            'id': last_post.id,
            'text': last_post.full_text,
            'in_reply_to_status_id': last_post.in_reply_to_status_id
        }
Beispiel #2
0
def mock_mention_replies_to_another_tweet(mock_tweepy, twitter_user):
    original_tweet = Status()
    original_tweet.id = 1
    original_tweet.full_text = "$AMZN is a great buy in 2021"

    reply_tweet = Status()
    reply_tweet.id = 2
    reply_tweet.text = "@stock_reminder remind me of this in 1 year"
    reply_tweet.user = twitter_user
    reply_tweet.in_reply_to_status_id = original_tweet.id

    mock_tweepy.return_value.mentions_timeline.return_value = [reply_tweet]
    mock_tweepy.return_value.get_status.return_value = original_tweet
    return mock_tweepy
Beispiel #3
0
def read_as_status(archivepath, since_id=None, max_id=None):
    api = API()

    since_id = since_id or 0
    max_id = max_id or float("inf")

    for tweet in archive.read_json(archivepath):

        try:
            tweet['created_at'] = fix_timeformat(tweet['created_at'])
        except AttributeError:
            tweet['created_at'] = fix_timeformat(tweet['timestamp'])

        if tweet.get('retweeted_status'):
            tweet['retweeted_status']['created_at'] = fix_timeformat(
                tweet['retweeted_status']['created_at'])

        try:
            i = int(tweet['id_str'])
        except AttributeError:
            i = int(tweet['tweet_id'])
            tweet['id_str'] = tweet['tweet_id']

        if i > since_id and i < max_id:
            yield Status.parse(api, tweet)
Beispiel #4
0
    def test_shortreply_with_mentions(self):
        """
        A reply from chromachipper uses the short reply where necessary.
        """

        # This will cause chromachipper to create a reply with short reply text.
        status_morethan140_char_obj = copy.deepcopy(self.status_base_obj)
        status_morethan140_char_obj['text'] = u'@chromachipper #000 @abcdefghijklmno @pqrstuvwxyzabcd @efghijklmnopqur @stuvqxyzabcdefg @hijklmnopqrstuv @wxyzabcdefghijk @ashortreply'
        status_morethan140_char_obj['entities']['user_mentions'] = [
            {u'id': 00000000, u'indices': [0, 14], u'id_str': u'00000000', u'screen_name': u'chromachipper', u'name': u'Chroma Chipper'},
            {u'id': 11111111, u'indices': [85, 96], u'id_str': u'11111111', u'screen_name': u'abcdefghijklmno', u'name': u'abcdefghijklmno'},
            {u'id': 22222222, u'indices': [85, 96], u'id_str': u'22222222', u'screen_name': u'pqrstuvwxyzabcd', u'name': u'pqrstuvwxyzabcd'},
            {u'id': 33333333, u'indices': [85, 96], u'id_str': u'33333333', u'screen_name': u'efghijklmnopqur', u'name': u'efghijklmnopqur'},
            {u'id': 44444444, u'indices': [85, 96], u'id_str': u'44444444', u'screen_name': u'stuvqxyzabcdefg', u'name': u'stuvqxyzabcdefg'},
            {u'id': 55555555, u'indices': [85, 96], u'id_str': u'55555555', u'screen_name': u'hijklmnopqrstuv', u'name': u'hijklmnopqrstuv'},
            {u'id': 66666666, u'indices': [85, 96], u'id_str': u'66666666', u'screen_name': u'wxyzabcdefghijk', u'name': u'wxyzabcdefghijk'},
            {u'id': 77777777, u'indices': [85, 96], u'id_str': u'77777777', u'screen_name': u'ashortreply', u'name': u'ashortreply'},
        ]
        status_140_char = Status.parse(api=API(), json=status_morethan140_char_obj)

        listener = ChromachipStreamListener(api=self.api, twitter_id=00000000)
        listener.on_status(status_140_char)

        self.assertEqual(self.api.last_status, u"@IWantToGift2U sent this! @abcdefghijklmno @pqrstuvwxyzabcd @efghijklmnopqur @stuvqxyzabcdefg @hijklmnopqrstuv @wxyzabcdefghijk @ashortreply")
        self.assertEqual(len(self.api.last_status), 140)
Beispiel #5
0
def mock_mention_replies_to_extended_tweet(mock_tweepy, twitter_user):
    original_tweet = Status()
    original_tweet.id = 1
    original_tweet.full_text = ("You wouldn’t believe it, but right now, "
                                "I think we’re looking at potentially 100% "
                                "stock returns in ~3 years for:\n\n1. $AMZN "
                                "at $3,055\n2. $TSLA at $661\n3. $JNJ at "
                                "$160\n Zero interest in debating!")

    reply_tweet = Status()
    reply_tweet.id = 2
    reply_tweet.text = "@stock_reminder remind me of this in 3 years"
    reply_tweet.user = twitter_user
    reply_tweet.in_reply_to_status_id = original_tweet.id

    mock_tweepy.return_value.mentions_timeline.return_value = [reply_tweet]
    mock_tweepy.return_value.get_status.return_value = original_tweet
    return mock_tweepy
Beispiel #6
0
def status(twitter_user):
    tweet = Status()
    tweet.id = 1
    tweet.text = "Price of $AMZN in 3 months."
    tweet.user = twitter_user
    tweet.in_reply_to_status_id = None
    return tweet
Beispiel #7
0
    def setUp(self):
        status_base_obj = {
            u'contributors': None,
            u'truncated': False,
            u'in_reply_to_status_id': None,
            u'id': 487394915643842560,
            u'retweeted': False,
            u'entities': {
                u'symbols': [],
                u'hashtags': [{u'indices': [97, 104], u'text': u'00b9f1'}],
                u'urls': []},
            u'in_reply_to_screen_name': u'chromachipper',
            u'id_str': u'487394915643842560',
            u'in_reply_to_user_id': 00000000,
            u'user': {
                u'id': 725633,
                u'id_str': u'725633',
                u'screen_name': u'IWantToGive',
                u'name': u'I am a Giver',
            },
        }
        status_with_mentions_obj = copy.deepcopy(status_base_obj)
        status_with_mentions_obj['text'] = u'@chromachipper what about if I mention another account. Can I gift someone a colour? @GiftToMe #00b9f1'
        status_with_mentions_obj['entities']['user_mentions'] = [
            {u'id': 00000000, u'indices': [0, 14], u'id_str': u'00000000', u'screen_name': u'chromachipper', u'name': u'Chroma Chipper'},
            {u'id': 11111111, u'indices': [85, 96], u'id_str': u'11111111', u'screen_name': u'GiftToMe', u'name': u'Gift Tome'},
            {u'id': 22222222, u'indices': [85, 96], u'id_str': u'22222222', u'screen_name': u'GiftToMeToo', u'name': u'Gift Tometoo'}
        ]

        status_without_mentions_obj = copy.deepcopy(status_base_obj)
        status_without_mentions_obj['text'] = u'@chromachipper send me a Chroma Chip! #00b9f1'
        status_without_mentions_obj['entities']['user_mentions'] = [
            {u'id': 00000000, u'indices': [0, 14], u'id_str': u'00000000', u'screen_name': u'chromachipper', u'name': u'Chroma Chipper'}
        ]

        self.api = MockAPI()
        self.status_with_mentions = Status.parse(api=API(), json=status_with_mentions_obj)
        self.status_without_mentions = Status.parse(api=API(), json=status_without_mentions_obj)
Beispiel #8
0
    def on_data(self, data):
        if not data:
            return
        json_data = json.loads(data)
        api = self.__api
        
        if not isinstance(json_data, dict):
            return

        if 'event' in json_data:
            if json_data['event']=='follow':
                target = User.parse(api, json_data["target"])
                source = User.parse(api, json_data["source"])
                if self.on_follow(target, source) is False:
                    return False
            elif json_data['event']=='favorite':
                target = Status.parse(api, json_data['target_object'])
                source = User.parse(api, json_data['source'])
                if self.on_favorite(target, source) is False:
                    return False
            elif json_data['event']=='unfavorite':
                target = Status.parse(api, json_data['target_object'])
                source = User.parse(api, json_data['source'])
                if self.on_unfavorite(target, source) is False:
                    return False
        elif 'delete' in json_data:
            delete = json_data['delete']['status']
            if self.on_delete(delete['id'], delete['user_id']) is False:
                return False
        elif 'in_reply_to_status_id' in json_data:
            status = Status.parse(api, json_data)
            if self.on_status(status) is False:
                return False
        elif 'limit' in json_data:
            if self.on_limit(json_data['limit']['track']) is False:
                return False
Beispiel #9
0
    def init_on_load(self):
        self._words = []
        self._keywords = []
        self._hashtags = []

        self.status = None
        if self.raw_json:
            self.status = Status.parse(None, json.loads(self.raw_json))
            
        if self.status:
            self.id = self.status.id
            self.author_name = self.status.author.screen_name
            self.create_time = self.status.created_at
            self._hashtags = ['#%s' % h['text'].lower() for h in self.status.entities.get('hashtags', [])]
            self._words = self.status.text.split(' ')
Beispiel #10
0
    def on_data(self, data):
        if not data:
            return
        json_data = json.loads(data)
        api = self.__api

        if not isinstance(json_data, dict):
            return

        if 'event' in json_data:
            if json_data['event'] == 'follow':
                target = User.parse(api, json_data["target"])
                source = User.parse(api, json_data["source"])
                if self.on_follow(target, source) is False:
                    return False
            elif json_data['event'] == 'favorite':
                target = Status.parse(api, json_data['target_object'])
                source = User.parse(api, json_data['source'])
                if self.on_favorite(target, source) is False:
                    return False
            elif json_data['event'] == 'unfavorite':
                target = Status.parse(api, json_data['target_object'])
                source = User.parse(api, json_data['source'])
                if self.on_unfavorite(target, source) is False:
                    return False
        elif 'delete' in json_data:
            delete = json_data['delete']['status']
            if self.on_delete(delete['id'], delete['user_id']) is False:
                return False
        elif 'in_reply_to_status_id' in json_data:
            status = Status.parse(api, json_data)
            if self.on_status(status) is False:
                return False
        elif 'limit' in json_data:
            if self.on_limit(json_data['limit']['track']) is False:
                return False
Beispiel #11
0
    def on_data(self, data):
        """Called when raw data is received from connection.

        Override this method if you wish to manually handle
        the stream data. Return False to stop stream and close connection.
        """

        if 'in_reply_to_status_id' in data and '"lang":"en"' in data:
            status = Status.parse(self.api, json.loads(data))
            if self.on_status(status) is False:
                return False
        elif 'delete' in data:
            delete = json.loads(data)['delete']['status']
            if self.on_delete(delete['id'], delete['user_id']) is False:
                return False
        elif 'limit' in data:
            if self.on_limit(json.loads(data)['limit']['track']) is False:
                return False
Beispiel #12
0
    def on_data(self, data):
        """Called when raw data is received from connection.

        Override this method if you wish to manually handle
        the stream data. Return False to stop stream and close connection.
        """

        if 'in_reply_to_status_id' in data and '"lang":"en"' in data:
            status = Status.parse(self.api, json.loads(data))
            if self.on_status(status) is False:
                return False
        elif 'delete' in data:
            delete = json.loads(data)['delete']['status']
            if self.on_delete(delete['id'], delete['user_id']) is False:
                return False
        elif 'limit' in data:
            if self.on_limit(json.loads(data)['limit']['track']) is False:
                return False
    def process_twitter_data(
            status: Status,
            twitter_credentials: dict,
            add_sentiment: bool = True,
            add_bot_analysis: bool = True) -> TwitterDataOutput:
        output: TwitterDataOutput = TwitterDataOutput(status={}, user={})
        try:

            # 1. Retrieve status document
            status_doc: StatusDoc = TwitterDataProcessor.process_status(
                status=status)
            response_sent: SentimentAnalysisOutput = SentimentAnalysisOutput()

            # 2. Add additional parameters related to sentiment analysis
            if add_sentiment:
                if len(status_doc.text) >= TwitterDataProcessor.min_char:
                    response_sent: SentimentAnalysisOutput = TwitterDataProcessor.process_sentiment_analysis(
                        doc=status_doc.text)
            status_doc.sentiment_analysis: dict = response_sent.__dict__

            # 3. Get the user
            user: User = status.__getattribute__("user")
            user_doc: UserAccountDoc = TwitterDataProcessor.process_user(
                user=user)

            # 4. Add additional params
            if add_bot_analysis:
                response_botometer_analysis: BotometerAnalysisOutput = TwitterDataProcessor.process_botometer_analysis(
                    user_id=user_doc.id,
                    twitter_credentials=twitter_credentials)
                user_doc.botometer_analysis: dict = response_botometer_analysis.__dict__

            # 5. Get the output
            output: TwitterDataOutput = TwitterDataOutput(
                status=status_doc.__dict__, user=user_doc.__dict__)
        except Exception as e:
            logger.error(e)
        return output
Beispiel #14
0
    def on_status(self, status: Status):

        logger.info(f"1. Loading Status with ID {status.__getattribute__('id')}")

        # 1. Check whether the status is already in the storage
        non_exists: bool = self.check_data_in_storage(
            entity_id=status.__getattribute__('id'),
            storage=self.storage,
            collection_name=self.collection_names.get("status"),
            identifier_key="id")

        if non_exists:
            # 3. Process Tweets an Users
            logger.info(f"2. Pre-processing Status with ID {status.__getattribute__('id')}")
            data: TwitterDataOutput = self.process_status(
                status=status, add_sentiment=self.add_sentiment,
                add_bot_analysis=self.add_bot_analysis)

            logger.info(f"3. Storing Status with ID {status.__getattribute__('id')} in {self.storage.title()}")
            # 2. Storage data
            self.storage_data(data=data, collection_names=self.collection_names,
                              storage=self.storage, mongodb_connector=self.mongodb_connector,
                              elasticsearch_connector=self.elasticsearch_connector,
                              identifier_key=self.identifier_key)
Beispiel #15
0
def read_as_status(archivepath, since_id=None, max_id=None):
    api = API()

    since_id = since_id or 0
    max_id = max_id or float("inf")

    for tweet in archive.read_json(archivepath):

        try:
            tweet['created_at'] = fix_timeformat(tweet['created_at'])
        except AttributeError:
            tweet['created_at'] = fix_timeformat(tweet['timestamp'])

        if tweet.get('retweeted_status'):
            tweet['retweeted_status']['created_at'] = fix_timeformat(tweet['retweeted_status']['created_at'])

        try:
            i = int(tweet['id_str'])
        except AttributeError:
            i = int(tweet['tweet_id'])
            tweet['id_str'] = tweet['tweet_id']

        if i > since_id and i < max_id:
            yield Status.parse(api, tweet)
Beispiel #16
0
def status(twitter_user):
    tweet = Status()
    tweet.id = 1
    tweet.text = "Price of $AMZN in 3 months."
    tweet.user = twitter_user
    return tweet
 def test_is_not_older_than_2_weeks(self):
     tweet = Status()
     tweet.created_at = datetime.now()
     self.assertFalse(is_older_than_2_weeks(tweet))
Beispiel #18
0
 def post_tweet(self, text: str) -> Status:
     log(f"Fake Tweet Sent: \"{text}\"")
     status = Status()
     status._json = {"text": f"{text}"}
     return status
Beispiel #19
0
 def get_user_timeline(self, id: str) -> List[Status]:
     status_1 = Status()
     status_1._json = {"text": "tweet_1"}
     status_2 = Status()
     status_2._json = {"text": "tweet_2"}
     return [status_2, status_1]
Beispiel #20
0
def assign_full_text_to_text(t: tweepy.Status):
    """Copy full_text attribute to text."""
    t.text = t.full_text
    t._json['text'] = t._json['full_text']
    return t
Beispiel #21
0
def delete(tweet: Status):
    tweet.destroy()
Beispiel #22
0
def status_from_dict(d: dict) -> Status:
    """Turn a dict into a Status."""
    return Status().parse(None, d)
 def test_is_older_than_2_weeks(self):
     tweet = Status()
     tweet.created_at = datetime(day=6, month=10, year=1975)
     self.assertTrue(is_older_than_2_weeks(tweet))
Beispiel #24
0
def __create_fake_user_timeline() -> List[Status]:
    status_1 = Status()
    status_1._json = {"text": "tweet_1 1. #"}
    status_2 = Status()
    status_2._json = {"text": "tweet_2 2. #"}
    return [status_2, status_1]
Beispiel #25
0
 def retweet(tweet: tweepy.Status) -> None:
     """Retweet a tweet."""
     try:
         tweet.retweet()
     except tweepy.TweepError as e:
         utils.log_tweepy_error(e)
Beispiel #26
0
def status(twitter_user):
    tweet = Status()
    tweet.id = 1
    tweet.text = "What is the current price of $BABA?"
    tweet.user = twitter_user
    return tweet
Beispiel #27
0
 def like(tweet: tweepy.Status) -> None:
     """Like a tweet."""
     try:
         tweet.favorite()
     except tweepy.TweepError as e:
         utils.log_tweepy_error(e)
Beispiel #28
0
def status_with_multiple_stocks(twitter_user):
    tweet = Status()
    tweet.id = 1
    tweet.text = "Remind me of $AMZN, $MSFT, $AAPL and $BABA in 3 months."
    tweet.user = twitter_user
    return tweet
Beispiel #29
0
def assign_full_text_to_text(t: tweepy.Status):
    """Copy full_text attribute to text."""
    t.text = t.full_text
    t._json['text'] = t._json['full_text']
    return t
Beispiel #30
0
def get_status(text, created_at=None):
    _json = {"text": text}
    if created_at:
        _json['created_at'] = created_at.strftime("%a %b %d %H:%M:%S %Y")
    return Status().parse(object, _json)