def tw_drop_followers(channel):
"""
When switching a channel to inactive, drop all the stored
followers from the channel from actual twitter tracking.
"""
from solariat_bottle.db.tracking import PostFilterStream
from solariat_bottle.db.channel.twitter import \
FollowerTrackingChannel, FollowerTrackingStatus
if isinstance(channel, FollowerTrackingChannel):
for fts in FollowerTrackingStatus.objects.find(channel=channel.id):
fts.update(set__followers_synced=0, set__sync_status='idle')
else:
channel.update(set__followers_synced=0, set__sync_status='idle')
stream = PostFilterStream.get()
if isinstance(channel, FollowerTrackingStatus):
main_channel = FollowerTrackingChannel.objects.get(channel.channel)
if main_channel.tracking_mode == 'Passive':
stream.untrack_channel_passive(main_channel,
channel.twitter_handle)
elif main_channel.tracking_mode == 'Active':
stream.untrack_channel(main_channel, channel.twitter_handle)
elif isinstance(channel, FollowerTrackingChannel):
if channel.tracking_mode == 'Passive':
stream.untrack_channel_passive(channel)
elif channel.tracking_mode == 'Active':
stream.untrack_channel(channel)
def save_followers(channel, data):
"""
TODO: PostFilterStream still has twitter_handle used, need
to refactor it to make it more generic to multiple sources
like fb/email/chat.
"""
from solariat_bottle.db.tracking import PostFilterStream
from solariat_bottle.db.channel.twitter import \
FollowerTrackingChannel, FollowerTrackingStatus
data = [str(i) for i in data]
stream = PostFilterStream.get()
if isinstance(channel, FollowerTrackingStatus):
main_channel = FollowerTrackingChannel.objects.get(channel.channel)
if main_channel.tracking_mode == 'Passive':
stream.track_passive(data, [main_channel],
twitter_handle=channel.twitter_handle)
else:
stream.track('USER_ID',
data, [main_channel],
twitter_handle=channel.twitter_handle)
else:
stream.track('USER_ID',
data, [channel],
twitter_handle=channel.twitter_handle)
def pre_save(self):
"Track/untrack twitter_handle"
from solariat_bottle.db.tracking import PostFilterStream
stream = PostFilterStream.get()
stream.untrack_channel(self)
if self.twitter_handle and self.status != 'Archived':
stream.track('USER_NAME', [self.twitter_handle], [self])
def _untrack_usernames(self, usernames=None):
usernames = usernames or self.usernames
from solariat_bottle.db.tracking import PostFilterStream
from solariat_bottle.utils.post import normalize_screen_name
stream = PostFilterStream.get()
stream.untrack('KEYWORD',
map(normalize_screen_name, usernames),
[self.inbound_channel],
langs=self.langs)
def create_by_user(self, user, **kw):
from solariat_bottle.db.tracking import PostFilterStream
channel = super(EnterpriseTwitterChannelManager,
self).create_by_user(user, **kw)
twitter_handle = kw.get('twitter_handle')
if twitter_handle:
stream = PostFilterStream.get()
stream.track('USER_NAME', [twitter_handle], [channel])
return channel
def del_skipword(self, skipword):
" del skipword "
from solariat_bottle.db.tracking import PostFilterStream
_skipwords = set(self.skipwords)
_skipwords.discard(skipword)
self.skipwords = list(_skipwords)
self.update(pull__skipwords=skipword)
stream = PostFilterStream.get()
stream.untrack('SKIPWORD', [LingualToken.unlangify(skipword)], [self],
langs=self.__get_token_langs(skipword))
def del_keyword(self, keyword):
" del keyword "
from solariat_bottle.db.tracking import PostFilterStream
_keywords = set(self.keywords)
_keywords.discard(keyword)
self.keywords = list(_keywords)
self.update(pull__keywords=keyword)
stream = PostFilterStream.get()
stream.untrack('KEYWORD', [LingualToken.unlangify(keyword)], [self],
langs=self.__get_token_langs(keyword))
def del_username(self, username):
" del username "
from solariat_bottle.db.tracking import PostFilterStream
_usernames = set(self.usernames)
_usernames.discard(username)
self.usernames = list(_usernames)
self.update(pull__usernames=username)
stream = PostFilterStream.get()
stream.untrack('USER_NAME', [username], [self], langs=self.langs)
def safe_channels(filter_channel_map):
channel_id_filter_map = defaultdict(set)
for filter_id, (filter_type,
channel_refs) in filter_channel_map.iteritems():
for ref in channel_refs:
channel_id_filter_map[ref.id].add((filter_id, filter_type))
expected_channel_ids = channel_id_filter_map.keys()
channels = list(
Channel.objects.coll.find({
Channel.F.id: {
"$in": expected_channel_ids
},
Channel.F.status: {
"$in": ['Active', 'Interim']
}
}))
channels = map(Channel, channels)
active_channel_ids = set(ch.id for ch in channels)
missing_channels = set(expected_channel_ids) - active_channel_ids
if missing_channels:
from solariat.db.abstract import DBRef
models = {ACTIVE: PostFilterEntry, PASSIVE: PostFilterEntryPassive}
for channel_id in missing_channels:
channel_ref = DBRef('Channel', channel_id)
for filter_id, filter_type in channel_id_filter_map[
channel_id]:
model = models[filter_type]
logger.warning("Channel pulled from %s(%s): %s" %
(model.__name__, filter_id, channel_id))
model.objects.coll.update(
{"_id": filter_id},
{"$pull": {
model.F('channels'): channel_ref
}})
PostFilterStream.refresh_stats()
PostFilterEntry.objects.remove(channels=[])
return channels
def __fix_tokens_langs(self, langs, action):
keywords = [
key for key in self.keywords if not is_token_lang_adopted(key)
]
skipwords = [
skip for skip in self.skipwords if not is_token_lang_adopted(skip)
]
from solariat_bottle.db.tracking import PostFilterStream
stream = PostFilterStream.get()
if hasattr(stream, action):
getattr(stream, action)('KEYWORD', keywords, [self], langs=langs)
getattr(stream, action)('SKIPWORD', skipwords, [self], langs=langs)
def add_username(self, username):
" add username "
# self.sync_contacts(user=username, platform='Twitter')
from solariat_bottle.db.tracking import PostFilterStream
_usernames = set(self.usernames)
_usernames.add(username)
self.usernames = list(_usernames)
self.update(addToSet__usernames=username)
if self.status in {'Active', 'Interim'}:
stream = PostFilterStream.get()
stream.track('USER_NAME', [username], [self], langs=self.langs)
def remove_langs(self, langs):
super(TwitterServiceChannel, self).remove_langs(langs)
from solariat_bottle.db.tracking import PostFilterStream
from solariat_bottle.utils.post import normalize_screen_name
usernames = self.usernames
stream = PostFilterStream.get()
stream.untrack('KEYWORD',
map(normalize_screen_name, usernames),
[self.inbound_channel],
langs=langs)
stream.untrack('USER_NAME',
usernames, [self.outbound_channel],
langs=langs)
def on_active(self):
" run this handler when channel activated "
# It's a temporary status which will be overwritten
# by datasift_sync2 script in 1.5 mins (at max)
super(KeywordTrackingChannel, self).on_active()
from solariat_bottle.db.tracking import PostFilterStream
stream = PostFilterStream.get()
for keyword in self.keywords:
stream.track('KEYWORD', [LingualToken.unlangify(keyword)], [self],
langs=self.__get_token_langs(keyword))
for skipword in self.skipwords:
stream.track('SKIPWORD', [LingualToken.unlangify(skipword)],
[self],
langs=self.__get_token_langs(skipword))
def add_skipword(self, skipword):
" add skipword "
if not self.__is_token_valid(skipword, self.skipwords):
return False
from solariat_bottle.db.tracking import PostFilterStream
_skipwords = set(self.skipwords)
_skipwords.add(skipword)
self.skipwords = list(_skipwords)
self.update(addToSet__skipwords=skipword)
if self.status in ('Active', 'Interim'):
stream = PostFilterStream.get()
stream.track('SKIPWORD', [LingualToken.unlangify(skipword)],
[self],
langs=self.__get_token_langs(skipword))
return True
def set_allowed_langs(self, langs, clear_previous=False):
current_langs = set(self.langs)
super(TwitterServiceChannel,
self).set_allowed_langs(langs, clear_previous)
from solariat_bottle.db.tracking import PostFilterStream
from solariat_bottle.utils.post import normalize_screen_name
new_langs = set(langs) - current_langs
usernames = self.usernames
stream = PostFilterStream.get()
stream.track('KEYWORD',
map(normalize_screen_name, usernames),
[self.inbound_channel],
langs=new_langs)
stream.track('USER_NAME',
usernames, [self.outbound_channel],
langs=new_langs)
def check_post_filter_entries(self, log_level=None, track_missing=False):
from solariat_bottle.db.tracking import PostFilterStream
if log_level:
log = getattr(LOGGER, log_level)
stream = PostFilterStream.get()
tests_results = []
for (filter_type, entries, channel, langs) in self.tracked_entities():
tracked = stream.tracking_state(filter_type, entries, [channel],
langs)
all_tracked = all(is_tracked for (_, _, is_tracked) in tracked)
tests_results.append(all_tracked)
if not all_tracked:
if log_level and log:
log("Missing PostFilterEntries ({}) for channel '{}'\n"
"Details: {}".format(filter_type, channel, tracked))
# track missing PostFilterEntries
if track_missing:
stream.track(filter_type, entries, [channel], langs=langs)
return all(tests_results)
def test_contacts_added_when_post_created(self):
user_profile = UserProfile.objects.upsert(
"Twitter", dict(screen_name="screen_name1"))
from solariat_bottle.db.tracking import PostFilterStream
stream = PostFilterStream.get()
#setup accounts
account = Account.objects.get_or_create(name="Test Account")
account2 = Account.objects.get_or_create(name="Test Account2")
self.user.account = account
self.user.save()
account2.add_user(self.user)
from ..db.channel.twitter import TwitterServiceChannel
service_channel1 = TwitterServiceChannel.objects.create_by_user(
self.user, account=account, title='Service')
service_channel2 = TwitterServiceChannel.objects.create_by_user(
self.user, account=account2, title='Service')
self.assertFalse(user_profile.screen_name in
account.get_contact_channel("Twitter").usernames)
self.assertFalse(user_profile.screen_name in
account2.get_contact_channel("Twitter").usernames)
channels = [service_channel1, service_channel2]
#create inbound post from user_profile
self._create_db_post(channels=channels,
content="Test Post",
user_profile=user_profile,
url='https://twitter.com/fake/status/fake.status')
#test user_profile is in contact channels for both accounts
self.assertTrue(user_profile.screen_name in
account.get_contact_channel("Twitter").usernames)
self.assertTrue(user_profile.screen_name in
account2.get_contact_channel("Twitter").usernames)
def on_active(self):
super(UserTrackingChannel, self).on_active()
from solariat_bottle.db.tracking import PostFilterStream
stream = PostFilterStream.get()
stream.track('USER_NAME', self.usernames, [self], langs=self.langs)
def on_suspend(self):
super(KeywordTrackingChannel, self).on_suspend()
from solariat_bottle.db.tracking import PostFilterStream
stream = PostFilterStream.get()
stream.untrack_channel(self)
def setUp(self):
MainCase.setUp(self)
self.stream = PostFilterStream.get()