def test_multilang_skipword(self):
self.etc.set_allowed_langs(["es"], clear_previous=True)
self.etc.add_keyword("es__key1")
content = "This is test tweet for key1"
tracked_channels = get_tracked_channels(
'Twitter', dict(content=content, lang="es"))
self.assertEquals(len(tracked_channels), 1)
self.etc.add_skipword("key1")
tracked_channels = get_tracked_channels(
'Twitter', dict(content=content, lang="es"))
self.assertEquals(len(tracked_channels), 0)
self.etc.del_skipword("key1")
tracked_channels = get_tracked_channels(
'Twitter', dict(content=content, lang="es"))
self.assertEquals(len(tracked_channels), 1)
from solariat_bottle.db.tracking import PostFilterEntry
print list(PostFilterEntry.objects.coll.find())
self.etc.del_keyword("key1")
tracked_channels = get_tracked_channels(
'Twitter', dict(content=content, lang="es"))
print list(PostFilterEntry.objects.coll.find())
self.assertEquals(len(tracked_channels), 0)
def _test(delete_channel):
default_capacity = 5
PostFilter.spare_capacity.default = default_capacity
ch = Channel.objects.create(title='To be Deleted')
self.stream.track('KEYWORD', ['@john81'], [ch])
tracked_channels = get_tracked_channels(
'Twitter',
dict(content="Hi @John81 I have problems on my Win 8 Laptop"))
self.assertEqual([ch], tracked_channels)
self.assertEqual(PostFilterEntry.objects(channels=ch.id).count(), 1)
delete_channel(ch)
tracked_channels = get_tracked_channels(
'Twitter',
dict(content="Hi @John81 I have problems on my Win 8 Laptop"))
self.assertEqual([], tracked_channels)
self.assertEqual(PostFilterEntry.objects(channels=ch.id).count(), 0)
self.assertEqual(PostFilterEntry.objects().count(), 0)
post_filters = PostFilter.objects.find()
self.assertEqual(len(post_filters), 1)
self.assertEqual(post_filters[0].entry_count, 0)
self.assertEqual(post_filters[0].spare_capacity, default_capacity)
def test_skipword(self):
ktc1 = KeywordTrackingChannel.objects.create_by_user(
self.user, title='KeywordTrackingChannel', status='Active')
ktc1.add_keyword('iphone')
ktc1.add_skipword('iPad') # Note: the keyword in PostFilterEntry will be lowercase 'ipad'
ktc2 = KeywordTrackingChannel.objects.create_by_user(
self.user, title='KeywordTrackingChannel2', status='Active')
ktc2.add_keyword('iphone')
tracked_channels = get_tracked_channels('Twitter',
dict(user_name='user', user_id='1'),
dict(content='I have iphone but not ipad'))
self.assertEqual(len(tracked_channels), 1)
self.assertTrue(ktc1 not in tracked_channels)
self.assertTrue(ktc2 in tracked_channels)
# test keywords case-insensitivity
tracked_channels = get_tracked_channels('Twitter',
dict(user_name='user', user_id='1'),
dict(content='I have the iPhone but not iPAD'))
self.assertEqual(len(tracked_channels), 1)
self.assertTrue(ktc1 not in tracked_channels)
self.assertTrue(ktc2 in tracked_channels)
def test_matching_by_keyword(self):
self.ch1.add_keyword('key')
self.ch2.add_keyword('es__key')
content = "This is test tweet for key"
tracked_channels = get_tracked_channels(
'Twitter', dict(content=content, lang="es"))
self.assertEqual(len(tracked_channels), 2)
tracked_channels = get_tracked_channels(
'Twitter', dict(content=content, lang="en"))
self.assertEqual(len(tracked_channels), 1)
def test_lang_sepcific_keyword(self):
self.etc.set_allowed_langs(["es"], clear_previous=True)
self.etc.add_keyword("es__key1")
content = "This is test tweet for key1"
tracked_channels = get_tracked_channels(
'Twitter', dict(content=content, lang="es"))
self.assertEquals(len(tracked_channels), 1)
tracked_channels = get_tracked_channels('Twitter',
dict(content=content))
self.assertFalse(tracked_channels)
def test_catch_mentions(self):
self.stream.track('KEYWORD', ['@john81'], [self.channel])
tracked_channels = get_tracked_channels('Twitter',
dict(content="Hi @John81 I have problems on my Win 8 Laptop"))
self.assertTrue(self.channel in tracked_channels)
def test_incorrect_hashtag(self):
self.stream.track('KEYWORD', ['#foobarcmg'], [self.channel])
tracked_channels = get_tracked_channels('Twitter',
dict(content="This is a #foobarcmg# I hope you like it."))
self.assertFalse(self.channel in tracked_channels)
def test_catch_hashtag_by_hashtag(self):
self.stream.track('KEYWORD', ['#foobarcmg'], [self.channel])
tracked_channels = get_tracked_channels('Twitter',
dict(content="This is a #foobarcmg! I hope you like it."))
self.assertTrue(self.channel in tracked_channels)
def test_user_name_case(self):
self.stream.track('USER_NAME', ['winPhoneSupport'], [self.channel])
tracked_channels = get_tracked_channels('Twitter',
dict(content="Xbox Music cloud is cool",
user_profile=dict(user_name='WinPhoneSupport', user_id='193456251')))
self.assertTrue(self.channel in tracked_channels)
def test_lang_agnostic_keyword_support(self):
self.etc.set_allowed_langs(["en", "es"])
self.etc.add_keyword("key1")
content = "This is test tweet for key1"
tracked_channels = get_tracked_channels('Twitter',
dict(content=content))
self.assertEquals(len(tracked_channels), 1)
def test_complex_key(self):
self.stream.track(
'KEYWORD',
["can't stop a girl"],
[self.channel])
tracked_channels = get_tracked_channels('Twitter',
dict(content="I Can't stop a girl from shopping!!"))
self.assertTrue(self.channel in tracked_channels)
def test_undefined_lang(self):
"""Should not filter out twitter posts with undefined language"""
self.etc.set_allowed_langs(["en"], clear_previous=True)
self.etc.add_keyword("key1")
self.etc.add_username("test_user")
for content, verify in [("AAAAAA key1", self.assertTrue),
("AAAAA", self.assertFalse),
("@test_user AAAAA", self.assertTrue),
("test_user AAAAA", self.assertFalse)]:
tracked_channels = get_tracked_channels(
'Twitter', dict(content=content, lang="und"))
verify(tracked_channels)
def test_track_passive(self):
self.stream.track_passive(["123456"], [self.channel])
self.assertTrue(PostFilterEntryPassive.objects.count() == 1)
tracked_channels = get_tracked_channels('Twitter',
dict(content="I need a new laptop", user_profile=dict(user_name='user_name', user_id='123456')))
self.assertTrue(self.channel in tracked_channels)
self.stream.untrack_channel_passive(self.channel)
self.assertTrue(PostFilterEntryPassive.objects.count() == 0)
def _create_tracked_post(self, content, user_name):
user_profile = UserProfile.objects.upsert(
'Twitter', dict(screen_name=user_name, user_id=user_name))
lang = 'en'
tracked_channels = get_tracked_channels(
"Twitter",
dict(content=content, lang=lang, user_profile=dict(user_name=user_name,
user_id=user_name)))
post = self._create_db_post(
user_profile=user_profile,
channels=[str(c.id) for c in tracked_channels],
content=content,
lang=lang)
return post
def test_directed_to_handle(self):
"""DMs and tweets directed to a handle should all be received
in spite of the language detected"""
non_configured_lang = 'id'
channel_langs = ['fr', 'en', 'pt', 'ro', 'it', 'es']
assert non_configured_lang not in channel_langs
public_tweet = {u'contributors': None, u'truncated': False,
u'text': u'@jessy_makaroff I need a new laptop now',
u'is_quote_status': False, u'in_reply_to_status_id': None,
u'id': 740742270610247680, u'favorite_count': 0,
u'source': u'<a href="http://twitter.com" rel="nofollow">Twitter Web Client</a>',
u'retweeted': False, u'coordinates': None,
u'timestamp_ms': u'1465441694296', u'entities': {
u'user_mentions': [
{u'id': 4842980294, u'indices': [0, 15], u'id_str': u'4842980294',
u'screen_name': u'jessy_makaroff', u'name': u'Jessy Petroff'}], u'symbols': [],
u'hashtags': [], u'urls': []}, u'in_reply_to_screen_name': u'jessy_makaroff',
u'id_str': u'740742270610247680', u'retweet_count': 0,
u'in_reply_to_user_id': 4842980294, u'favorited': False,
u'user': {u'follow_request_sent': None,
u'profile_use_background_image': True,
u'default_profile_image': True, u'id': 4843024956,
u'verified': False,
u'profile_image_url_https': u'https://abs.twimg.com/sticky/default_profile_images/default_profile_0_normal.png',
u'profile_sidebar_fill_color': u'DDEEF6',
u'profile_text_color': u'333333', u'followers_count': 11,
u'profile_sidebar_border_color': u'C0DEED',
u'id_str': u'4843024956',
u'profile_background_color': u'F5F8FA',
u'listed_count': 0,
u'profile_background_image_url_https': u'',
u'utc_offset': None, u'statuses_count': 122,
u'description': None, u'friends_count': 12,
u'location': None, u'profile_link_color': u'2B7BB9',
u'profile_image_url': u'http://abs.twimg.com/sticky/default_profile_images/default_profile_0_normal.png',
u'following': None, u'geo_enabled': False,
u'profile_background_image_url': u'',
u'name': u'Tara Petroff', u'lang': u'en',
u'profile_background_tile': False,
u'favourites_count': 7, u'screen_name': u'PetroffTara',
u'notifications': None, u'url': None,
u'created_at': u'Mon Feb 01 05:16:16 +0000 2016',
u'contributors_enabled': False, u'time_zone': None,
u'protected': False, u'default_profile': True,
u'is_translator': False}, u'geo': None,
u'in_reply_to_user_id_str': u'4842980294', u'lang': non_configured_lang,
u'created_at': u'Thu Jun 09 03:08:14 +0000 2016',
u'filter_level': u'low', u'in_reply_to_status_id_str': None,
u'place': None}
direct_message = {u'sender_screen_name': u'PetroffTara', u'recipient_id_str': u'4842980294',
u'sender':
{u'follow_request_sent': False, u'profile_use_background_image': True,
u'contributors_enabled': False, u'id': 4843024956, u'verified': False,
u'profile_image_url_https': u'https://abs.twimg.com/sticky/default_profile_images/default_profile_0_normal.png',
u'profile_sidebar_fill_color': u'DDEEF6',
u'profile_text_color': u'333333', u'followers_count': 11,
u'profile_sidebar_border_color': u'C0DEED', u'location': None,
u'default_profile_image': True, u'id_str': u'4843024956',
u'is_translation_enabled': False, u'utc_offset': None,
u'statuses_count': 125, u'description': None, u'friends_count': 12,
u'profile_link_color': u'2B7BB9',
u'profile_image_url': u'http://abs.twimg.com/sticky/default_profile_images/default_profile_0_normal.png',
u'notifications': False, u'geo_enabled': False,
u'profile_background_color': u'F5F8FA',
u'profile_background_image_url': None, u'name': u'Tara Petroff',
u'lang': u'en', u'following': False,
u'profile_background_tile': False, u'favourites_count': 8,
u'screen_name': u'PetroffTara', u'url': None,
u'created_at': u'Mon Feb 01 05:16:16 +0000 2016',
u'profile_background_image_url_https': None, u'time_zone': None,
u'protected': False, u'default_profile': True,
u'is_translator': False, u'listed_count': 0}
, u'sender_id_str': u'4843024956', u'text': u'AAAA',
u'created_at': u'Wed Jun 15 13:04:51 +0000 2016', u'sender_id': 4843024956,
u'entities':
{u'symbols': [], u'user_mentions': [], u'hashtags': [], u'urls': []}
, u'recipient_id': 4842980294, u'id_str': u'743066741715046403',
u'recipient_screen_name': u'jessy_makaroff', u'recipient':
{u'follow_request_sent': False, u'profile_use_background_image': True,
u'contributors_enabled': False, u'id': 4842980294, u'verified': False,
u'profile_image_url_https': u'https://pbs.twimg.com/profile_images/694016704918081536/dj7tqhRm_normal.png',
u'profile_sidebar_fill_color': u'DDEEF6',
u'profile_text_color': u'333333', u'followers_count': 13,
u'profile_sidebar_border_color': u'C0DEED', u'location': None,
u'default_profile_image': False, u'id_str': u'4842980294',
u'is_translation_enabled': False, u'utc_offset': None,
u'statuses_count': 7868, u'description': None, u'friends_count': 20,
u'profile_link_color': u'2B7BB9',
u'profile_image_url': u'http://pbs.twimg.com/profile_images/694016704918081536/dj7tqhRm_normal.png',
u'notifications': False, u'geo_enabled': False,
u'profile_background_color': u'F5F8FA',
u'profile_background_image_url': None, u'name': u'Jessy Petroff',
u'lang': u'en', u'following': False,
u'profile_background_tile': False, u'favourites_count': 2,
u'screen_name': u'jessy_makaroff', u'url': None,
u'created_at': u'Mon Feb 01 04:35:17 +0000 2016',
u'profile_background_image_url_https': None, u'time_zone': None,
u'protected': False, u'default_profile': True,
u'is_translator': False, u'listed_count': 4}
, u'id': 743066741715046403}
from solariat_bottle.daemons.helpers import twitter_dm_to_post_dict, \
twitter_status_to_post_dict
public_tweet_data = twitter_status_to_post_dict(public_tweet)
direct_message_data = twitter_dm_to_post_dict(direct_message)
# Overriding language to guarantee it's not a language from the channel configuration.
# This makes get_language() bypass the internal language detection routine.
direct_message_data['lang'] = non_configured_lang
public_tweet_data['lang'] = non_configured_lang
sc = TwitterServiceChannel.objects.create_by_user(
self.user,
account=self.account,
title='TSC',
status='Active')
sc.set_allowed_langs(channel_langs)
sc.add_keyword('@jessy_makaroff')
sc.add_username('@jessy_makaroff')
etc = EnterpriseTwitterChannel.objects.create_by_user(
self.user,
account=self.account,
title='TA',
status='Active',
twitter_handle='jessy_makaroff')
for data in [public_tweet_data, direct_message_data]:
channels = get_tracked_channels('Twitter', data)
self.assertEqual(channels, [sc.inbound_channel])
def _test():
return get_tracked_channels(
'Twitter',
dict(content='RT: @aaa I like test_kwd',
user_profile=dict(user_name='test', user_id='1'),
twitter=dict(retweeted_status={"content": "I like test_kwd"})))
