def twitter_status_to_post_dict(data):
base_url = 'https://twitter.com/%s/statuses/%s'
if 'text' in data or 'full_text' in data:
extended_tweet = data.get('extended_tweet') or {}
is_retweet = 'retweeted_status' in data
post_fields = {'twitter': {'_wrapped_data': json.dumps(data)}}
post_fields['twitter'].update(
_source=TweetSource.TWITTER_PUBLIC_STREAM, _is_retweet=is_retweet)
if 'user' in data:
author = data['user']
content = data.get('full_text') or extended_tweet.get(
'full_text') or data.get('text')
else:
LOGGER.warn(u"Mis-formatted twitter data %s" % data)
return
def _get_tweet_lang(data):
if 'lang' in data and data['lang']:
return data['lang']
elif 'lang' in author:
return author['lang']
post_fields['twitter'].update(
_is_manual_retweet=not is_retweet and content.startswith('RT'))
post_fields['lang'] = _get_tweet_lang(data)
post_fields['user_profile'] = parse_user_profile(author)
post_fields['content'] = content
post_fields['url'] = base_url % (author['screen_name'], data['id_str'])
#twitter data used to link a post to possible thread
post_fields['twitter'].update(TweetParser()(data))
return post_fields
def get_user_profile(tw_json):
from solariat_bottle.daemons.twitter.parsers import parse_user_profile
from solariat_bottle.db.user_profiles.user_profile import UserProfile
try:
return UserProfile.objects.get_by_platform('Twitter', tw_json['screen_name'])
except UserProfile.DoesNotExist:
return UserProfile.objects.upsert('Twitter', profile_data=parse_user_profile(tw_json))
def twitter_dm_to_post_dict(dm):
""" Converts a Twitter direct message data structure (coming from the Twitter Stream API)
to a post-fields dict (for passing to a <PostCreator> or db.post.utils.factory_by_user)
"""
user_profile = parse_user_profile(dm['sender'])
post_fields = dict(
twitter={
'_wrapped_data': json.dumps(dm),
'_source': TweetSource.TWITTER_USER_STREAM,
},
user_profile=user_profile,
content=dm['text'] + ' @%s' % dm['recipient']['screen_name'],
message_type=1,
# the following is for tasks.get_tracked_channels which is going
# to be called by PostCreator because we do not provide channels here
direct_message=True,
recipient_handle=dm['recipient']['screen_name'],
sender_handle=dm['sender']['screen_name'],
)
post_fields['twitter'].update(DMTweetParser()(dm))
return post_fields
def _get_user_profile(author, klout):
result = parse_user_profile(author)
if klout:
result['klout_score'] = klout['score']
return result
def test_post_reply(self):
solariat1001 = {
u'follow_request_sent': None,
u'profile_use_background_image': True,
u'default_profile_image': True,
u'id': 2953886599,
u'verified': False,
u'profile_image_url_https':
u'https://abs.twimg.com/sticky/default_profile_images/default_profile_6_normal.png',
u'profile_sidebar_fill_color': u'DDEEF6',
u'profile_text_color': u'333333',
u'followers_count': 5,
u'profile_sidebar_border_color': u'C0DEED',
u'id_str': u'2953886599',
u'profile_background_color': u'C0DEED',
u'listed_count': 0,
u'profile_background_image_url_https':
u'https://abs.twimg.com/images/themes/theme1/bg.png',
u'utc_offset': None,
u'statuses_count': 191,
u'description': None,
u'friends_count': 5,
u'location': u'',
u'profile_link_color': u'0084B4',
u'profile_image_url':
u'http://abs.twimg.com/sticky/default_profile_images/default_profile_6_normal.png',
u'following': None,
u'geo_enabled': False,
u'profile_background_image_url':
u'http://abs.twimg.com/images/themes/theme1/bg.png',
u'name': u'Solariatw',
u'lang': u'en',
u'profile_background_tile': False,
u'favourites_count': 6,
u'screen_name': u'solariat1001',
u'notifications': None,
u'url': None,
u'created_at': u'Wed Dec 31 17:10:35 +0000 2014',
u'contributors_enabled': False,
u'time_zone': None,
u'protected': False,
u'default_profile': True,
u'is_translator': False
}
solariat1002 = {
u'follow_request_sent': None,
u'profile_use_background_image': True,
u'default_profile_image': True,
u'id': 2977841942,
u'verified': False,
u'profile_image_url_https':
u'https://abs.twimg.com/sticky/default_profile_images/default_profile_5_normal.png',
u'profile_sidebar_fill_color': u'DDEEF6',
u'profile_text_color': u'333333',
u'followers_count': 4,
u'profile_sidebar_border_color': u'C0DEED',
u'id_str': u'2977841942',
u'profile_background_color': u'C0DEED',
u'listed_count': 0,
u'profile_background_image_url_https':
u'https://abs.twimg.com/images/themes/theme1/bg.png',
u'utc_offset': -14400,
u'statuses_count': 56,
u'description': None,
u'friends_count': 5,
u'location': u'',
u'profile_link_color': u'0084B4',
u'profile_image_url':
u'http://abs.twimg.com/sticky/default_profile_images/default_profile_5_normal.png',
u'following': None,
u'geo_enabled': True,
u'profile_background_image_url':
u'http://abs.twimg.com/images/themes/theme1/bg.png',
u'name': u'Solariatr',
u'lang': u'en',
u'profile_background_tile': False,
u'favourites_count': 0,
u'screen_name': u'solariat1002',
u'notifications': None,
u'url': None,
u'created_at': u'Mon Jan 12 22:56:50 +0000 2015',
u'contributors_enabled': False,
u'time_zone': u'Eastern Time (US & Canada)',
u'protected': False,
u'default_profile': True,
u'is_translator': False
}
UserProfile.objects.upsert(
'Twitter', profile_data=parse_user_profile(solariat1001))
UserProfile.objects.upsert(
'Twitter', profile_data=parse_user_profile(solariat1002))
self.user.is_superuser = True
sc1 = self.setup_channel(keywords=[],
usernames=['@solariat1001'],
title='TS1')
sc2 = self.setup_channel(keywords=['new smart TV'],
usernames=['@solariat1002'],
title='TS2')
sc2.add_username(
'@solariat1001') # for routing all replies to sc2.outbound
ta1 = self.setup_outbound_channel('TA1', 'solariat1001')
ta2 = self.setup_outbound_channel('TA2', 'solariat1002')
self.user.set_outbound_channel(ta1)
self.user.current_account.set_outbound_channel(ta1)
stream = [{
u'contributors': None,
u'truncated': False,
u'text': u'I need a new smart TV tomorrow',
u'in_reply_to_status_id': None,
u'id': 598561105468850178,
u'favorite_count': 0,
u'source':
u'<a href="http://twitter.com" rel="nofollow">Twitter Web Client</a>',
u'retweeted': False,
u'coordinates': None,
u'timestamp_ms': u'1431543062928',
u'entities': {
u'user_mentions': [],
u'symbols': [],
u'trends': [],
u'hashtags': [],
u'urls': []
},
u'in_reply_to_screen_name': None,
u'id_str': u'598561105468850178',
u'retweet_count': 0,
u'in_reply_to_user_id': None,
u'favorited': False,
u'user': {
u'follow_request_sent': None,
u'profile_use_background_image': True,
u'default_profile_image': True,
u'id': 2750033348,
u'verified': False,
u'profile_image_url_https':
u'https://abs.twimg.com/sticky/default_profile_images/default_profile_6_normal.png',
u'profile_sidebar_fill_color': u'DDEEF6',
u'profile_text_color': u'333333',
u'followers_count': 4,
u'profile_sidebar_border_color': u'C0DEED',
u'id_str': u'2750033348',
u'profile_background_color': u'C0DEED',
u'listed_count': 0,
u'profile_background_image_url_https':
u'https://abs.twimg.com/images/themes/theme1/bg.png',
u'utc_offset': None,
u'statuses_count': 119,
u'description': None,
u'friends_count': 1,
u'location': u'',
u'profile_link_color': u'0084B4',
u'profile_image_url':
u'http://abs.twimg.com/sticky/default_profile_images/default_profile_6_normal.png',
u'following': None,
u'geo_enabled': False,
u'profile_background_image_url':
u'http://abs.twimg.com/images/themes/theme1/bg.png',
u'name': u'Solariat2014 Tester',
u'lang': u'en',
u'profile_background_tile': False,
u'favourites_count': 0,
u'screen_name': u'solariat2014',
u'notifications': None,
u'url': None,
u'created_at': u'Wed Aug 20 19:05:26 +0000 2014',
u'contributors_enabled': False,
u'time_zone': None,
u'protected': False,
u'default_profile': True,
u'is_translator': False
},
u'geo': None,
u'in_reply_to_user_id_str': None,
u'possibly_sensitive': False,
u'lang': u'en',
u'created_at': u'Wed May 13 18:51:02 +0000 2015',
u'filter_level': u'low',
u'in_reply_to_status_id_str': None,
u'place': None
}, {
u'contributors': None,
u'truncated': False,
u'text': u'@solariat2014 Reply from TA2 ^dna',
u'in_reply_to_status_id': 598561105468850178,
u'id': 598561193196818432,
u'favorite_count': 0,
u'source':
u'<a href="http://solariat.com" rel="nofollow">Solariat test</a>',
u'retweeted': False,
u'coordinates': None,
u'timestamp_ms': u'1431543083844',
u'entities': {
u'user_mentions': [{
u'id': 2750033348,
u'indices': [0, 13],
u'id_str': u'2750033348',
u'screen_name': u'solariat2014',
u'name': u'Solariat2014 Tester'
}],
u'symbols': [],
u'trends': [],
u'hashtags': [],
u'urls': []
},
u'in_reply_to_screen_name': u'solariat2014',
u'id_str': u'598561193196818432',
u'retweet_count': 0,
u'in_reply_to_user_id': 2750033348,
u'favorited': False,
u'user': solariat1002,
u'geo': None,
u'in_reply_to_user_id_str': u'2750033348',
u'possibly_sensitive': False,
u'lang': u'en',
u'created_at': u'Wed May 13 18:51:23 +0000 2015',
u'filter_level': u'low',
u'in_reply_to_status_id_str': u'598561105468850178',
u'place': None
}, {
u'contributors': None,
u'truncated': False,
u'text': u'My new smart TV is broken',
u'in_reply_to_status_id': None,
u'id': 598561297995800577,
u'favorite_count': 0,
u'source':
u'<a href="http://twitter.com" rel="nofollow">Twitter Web Client</a>',
u'retweeted': False,
u'coordinates': None,
u'timestamp_ms': u'1431543108830',
u'entities': {
u'user_mentions': [],
u'symbols': [],
u'trends': [],
u'hashtags': [],
u'urls': []
},
u'in_reply_to_screen_name': None,
u'id_str': u'598561297995800577',
u'retweet_count': 0,
u'in_reply_to_user_id': None,
u'favorited': False,
u'user': {
u'follow_request_sent': None,
u'profile_use_background_image': True,
u'default_profile_image': True,
u'id': 2750033348,
u'verified': False,
u'profile_image_url_https':
u'https://abs.twimg.com/sticky/default_profile_images/default_profile_6_normal.png',
u'profile_sidebar_fill_color': u'DDEEF6',
u'profile_text_color': u'333333',
u'followers_count': 4,
u'profile_sidebar_border_color': u'C0DEED',
u'id_str': u'2750033348',
u'profile_background_color': u'C0DEED',
u'listed_count': 0,
u'profile_background_image_url_https':
u'https://abs.twimg.com/images/themes/theme1/bg.png',
u'utc_offset': None,
u'statuses_count': 120,
u'description': None,
u'friends_count': 1,
u'location': u'',
u'profile_link_color': u'0084B4',
u'profile_image_url':
u'http://abs.twimg.com/sticky/default_profile_images/default_profile_6_normal.png',
u'following': None,
u'geo_enabled': False,
u'profile_background_image_url':
u'http://abs.twimg.com/images/themes/theme1/bg.png',
u'name': u'Solariat2014 Tester',
u'lang': u'en',
u'profile_background_tile': False,
u'favourites_count': 0,
u'screen_name': u'solariat2014',
u'notifications': None,
u'url': None,
u'created_at': u'Wed Aug 20 19:05:26 +0000 2014',
u'contributors_enabled': False,
u'time_zone': None,
u'protected': False,
u'default_profile': True,
u'is_translator': False
},
u'geo': None,
u'in_reply_to_user_id_str': None,
u'possibly_sensitive': False,
u'lang': u'en',
u'created_at': u'Wed May 13 18:51:48 +0000 2015',
u'filter_level': u'low',
u'in_reply_to_status_id_str': None,
u'place': None
}, {
u'contributors': None,
u'truncated': False,
u'text': u'@solariat2014 Reply from TA1 ^dna',
u'in_reply_to_status_id': 598561297995800577,
u'id': 598561354618802176,
u'favorite_count': 0,
u'source':
u'<a href="http://solariat.com" rel="nofollow">Solariat test</a>',
u'retweeted': False,
u'coordinates': None,
u'timestamp_ms': u'1431543122330',
u'entities': {
u'user_mentions': [{
u'id': 2750033348,
u'indices': [0, 13],
u'id_str': u'2750033348',
u'screen_name': u'solariat2014',
u'name': u'Solariat2014 Tester'
}],
u'symbols': [],
u'trends': [],
u'hashtags': [],
u'urls': []
},
u'in_reply_to_screen_name': u'solariat2014',
u'id_str': u'598561354618802176',
u'retweet_count': 0,
u'in_reply_to_user_id': 2750033348,
u'favorited': False,
u'user': solariat1001,
u'geo': None,
u'in_reply_to_user_id_str': u'2750033348',
u'possibly_sensitive': False,
u'lang': u'en',
u'created_at': u'Wed May 13 18:52:02 +0000 2015',
u'filter_level': u'low',
u'in_reply_to_status_id_str': u'598561297995800577',
u'place': None
}, {
u'contributors': None,
u'truncated': False,
u'text':
u"RT @SportsMania005: Over the next few days we are adding a new feature which is making our Streams compatible with Smart TV's so we will be\u2026",
u'in_reply_to_status_id': None,
u'id': 598562190078156800,
u'favorite_count': 0,
u'source':
u'<a href="http://twitter.com/download/android" rel="nofollow">Twitter for Android</a>',
u'retweeted': False,
u'coordinates': None,
u'timestamp_ms': u'1431543321519',
u'entities': {
u'user_mentions': [{
u'id': 2208881395,
u'indices': [3, 18],
u'id_str': u'2208881395',
u'screen_name': u'SportsMania005',
u'name': u'SportsMania'
}],
u'symbols': [],
u'trends': [],
u'hashtags': [],
u'urls': []
},
u'in_reply_to_screen_name': None,
u'id_str': u'598562190078156800',
u'retweet_count': 0,
u'in_reply_to_user_id': None,
u'favorited': False,
u'retweeted_status': {
u'contributors': None,
u'truncated': False,
u'text':
u"Over the next few days we are adding a new feature which is making our Streams compatible with Smart TV's so we will be confirming which...",
u'in_reply_to_status_id': None,
u'id': 598530764574203904,
u'favorite_count': 1,
u'source':
u'<a href="http://twitter.com" rel="nofollow">Twitter Web Client</a>',
u'retweeted': False,
u'coordinates': None,
u'entities': {
u'user_mentions': [],
u'symbols': [],
u'trends': [],
u'hashtags': [],
u'urls': []
},
u'in_reply_to_screen_name': None,
u'id_str': u'598530764574203904',
u'retweet_count': 2,
u'in_reply_to_user_id': None,
u'favorited': False,
u'user': {
u'follow_request_sent': None,
u'profile_use_background_image': False,
u'default_profile_image': False,
u'id': 2208881395,
u'verified': False,
u'profile_image_url_https':
u'https://pbs.twimg.com/profile_images/568481878542270464/236eDfg5_normal.png',
u'profile_sidebar_fill_color': u'000000',
u'profile_text_color': u'000000',
u'followers_count': 1361,
u'profile_sidebar_border_color': u'000000',
u'id_str': u'2208881395',
u'profile_background_color': u'000000',
u'listed_count': 12,
u'profile_background_image_url_https':
u'https://abs.twimg.com/images/themes/theme1/bg.png',
u'utc_offset': 3600,
u'statuses_count': 63,
u'description': None,
u'friends_count': 343,
u'location': u'Romania',
u'profile_link_color': u'DD2E44',
u'profile_image_url':
u'http://pbs.twimg.com/profile_images/568481878542270464/236eDfg5_normal.png',
u'following': None,
u'geo_enabled': False,
u'profile_banner_url':
u'https://pbs.twimg.com/profile_banners/2208881395/1424371546',
u'profile_background_image_url':
u'http://abs.twimg.com/images/themes/theme1/bg.png',
u'name': u'SportsMania',
u'lang': u'en',
u'profile_background_tile': False,
u'favourites_count': 102,
u'screen_name': u'SportsMania005',
u'notifications': None,
u'url': u'http://SportsMania.eu',
u'created_at': u'Fri Nov 22 12:18:31 +0000 2013',
u'contributors_enabled': False,
u'time_zone': u'Casablanca',
u'protected': False,
u'default_profile': False,
u'is_translator': False
},
u'geo': None,
u'in_reply_to_user_id_str': None,
u'possibly_sensitive': False,
u'lang': u'en',
u'created_at': u'Wed May 13 16:50:29 +0000 2015',
u'filter_level': u'low',
u'in_reply_to_status_id_str': None,
u'place': None
},
u'user': {
u'follow_request_sent': None,
u'profile_use_background_image': True,
u'default_profile_image': False,
u'id': 315963404,
u'verified': False,
u'profile_image_url_https':
u'https://pbs.twimg.com/profile_images/551095536871546880/ub3RZJC0_normal.jpeg',
u'profile_sidebar_fill_color': u'DDEEF6',
u'profile_text_color': u'333333',
u'followers_count': 156,
u'profile_sidebar_border_color': u'C0DEED',
u'id_str': u'315963404',
u'profile_background_color': u'C0DEED',
u'listed_count': 0,
u'profile_background_image_url_https':
u'https://abs.twimg.com/images/themes/theme1/bg.png',
u'utc_offset': None,
u'statuses_count': 1258,
u'description': None,
u'friends_count': 380,
u'location': u'Workington',
u'profile_link_color': u'0084B4',
u'profile_image_url':
u'http://pbs.twimg.com/profile_images/551095536871546880/ub3RZJC0_normal.jpeg',
u'following': None,
u'geo_enabled': False,
u'profile_banner_url':
u'https://pbs.twimg.com/profile_banners/315963404/1416002538',
u'profile_background_image_url':
u'http://abs.twimg.com/images/themes/theme1/bg.png',
u'name': u'shaun mandale',
u'lang': u'en',
u'profile_background_tile': False,
u'favourites_count': 995,
u'screen_name': u'S_mandale',
u'notifications': None,
u'url': None,
u'created_at': u'Sun Jun 12 18:40:55 +0000 2011',
u'contributors_enabled': False,
u'time_zone': None,
u'protected': False,
u'default_profile': True,
u'is_translator': False
},
u'geo': None,
u'in_reply_to_user_id_str': None,
u'possibly_sensitive': False,
u'lang': u'en',
u'created_at': u'Wed May 13 18:55:21 +0000 2015',
u'filter_level': u'low',
u'in_reply_to_status_id_str': None,
u'place': None
}]
self.start_bot()
for post in stream:
self.send_event('on_data', post)
self.wait_bot() # wait to preserve post order
# test both posts in sc2.inbound are replied
self.assertEqual(TwitterPost.objects(channels=sc2.inbound).count(), 2)
self.assertEqual(TwitterPost.objects(channels=sc2.outbound).count(), 2)
for post in TwitterPost.objects(channels=sc2.inbound):
self.assertEqual(post.channel_assignments[str(sc2.inbound)],
'replied')
for reply in TwitterPost.objects(channels=sc2.outbound):
self.assertIn(sc2.inbound, reply.parent.channels)
self.stop_bot()
def get_params(up_data):
if platform == 'Twitter':
from solariat_bottle.daemons.twitter.parsers import parse_user_profile
return parse_user_profile(up_data)
return {}
def upsert_user_profile(data):
from solariat_bottle.daemons.twitter.parsers import parse_user_profile
from solariat_bottle.db.user_profiles.user_profile import UserProfile
return UserProfile.objects.upsert('Twitter', parse_user_profile(data))