def tw_unfollow(channel, user_profile, silent_ex=False):
"""
Task for unfollowing a twitter screen name.
:param channel: Used to grab the twitter credentials we are using for the unfollow action.
:param screen_name: The twitter screen name we are going to unfollow
:param silent_ex: Optional, if true any exceptions will just be silently ignored
"""
from solariat_bottle.settings import get_var, LOGGER
from solariat_bottle.db.user_profiles.user_profile import UserProfile, get_brand_profile_id, get_native_user_id_from_channel
result = {}
if not get_var('ON_TEST') and get_var('APP_MODE') == 'prod':
from solariat_bottle.utils.oauth import get_twitter_oauth_credentials
(consumer_key, consumer_secret, _, access_token_key,
access_token_secret) = get_twitter_oauth_credentials(channel)
try:
result = tw_destroy_friendship(
consumer_key=consumer_key,
consumer_secret=consumer_secret,
access_token_key=access_token_key,
access_token_secret=access_token_secret,
screen_name=user_profile.screen_name)
except Exception, ex:
LOGGER.error("tasks.tw_unfollow: " + str(ex))
if silent_ex:
result = dict(error=str(ex))
else:
raise
def generate_shortened_url(long_url, access_token=False):
"""Generate bitly url, see
code.google.com/p/bitly-api/wiki/ApiDocumentation#/v3/shorten
"""
params = dict(longUrl=long_url)
if access_token:
params['access_token'] = access_token
else:
params['login'] = get_var('BITLY_LOGIN')
params['apiKey'] = get_var('BITLY_KEY')
if get_var('BITLY_DOMAIN'):
params['domain'] = get_var('BITLY_DOMAIN')
url = "%sshorten?%s" % (get_var('BITLY_BASE_URL'),
urllib.urlencode(params))
resp = requests.get(url)
if not resp.status_code == 200:
raise RuntimeError("Could not get shorten url, %s" % resp.headers)
result = resp.json()
try:
return result['data']['url']
except (KeyError, TypeError):
raise RuntimeError("Could not get shorten %s, %s" % (long_url, result))
def __init__(self):
self.INDEX_NAME = get_var(
'CHANNEL_FILTER_ITEM_INDEX_NAME') or 'channelfilteritems'
self.DOCUMENT_NAME = get_var(
'CHANNEL_FILTER_ITEM_DOCUMENT_NAME') or 'channelfilteritem'
ElasticCollection.__init__(self)
def tearDown(self):
if get_var('TEST_STAT_RESET', False):
to_date = now()
from_date = to_date - timedelta(days=90)
exceptions = get_var('EXCEPTED_RESET_CASES', [])
test_name = self.__class__.__name__ + '.' + self._testMethodName
if test_name not in exceptions:
try:
for account in Account.objects():
do_it(test_mode=True, raise_on_diffs=True, account_name=account.name,
to_date=to_date, from_date=from_date)
except Exception, ex:
self.fail(ex)
def clear_subscriptions():
"""
Clear all subscriptions we have on datasift for this given host domain.
"""
params = {
'username': get_var('DATASIFT_USERNAME'),
'api_key': get_var('DATASIFT_API_KEY')
}
existing_subs = list_subscriptions()
for subscription in existing_subs:
params['id'] = subscription['id']
requests.get(
get_var("DATASIFT_BASE_URL") + "/v1/push/delete",
urllib.urlencode(params))
def send_notification_for_team(subject, body=None):
suffix = "| host: '%s' at %s'" % (gethostname(), datetime.now())
subject = subject + suffix
if body is None:
body = subject
with app.app_context():
msg = Message(subject=subject,
sender=get_var("MAIL_DEFAULT_SENDER"),
recipients=get_var("SCRIPTS_EMAIL_LIST"),
body=body)
if get_var('APP_MODE') == 'prod':
mail.send(msg)
else:
LOGGER.info(msg.subject)
LOGGER.info(msg.body)
def purge_channel_stats(channel):
days = get_var('CHANNEL_STATS_KEEP_DAYS')
start_date = datetime(year=2012, month=1, day=1)
end_date = now() - timedelta(days=days)
# end_date = datetime(year=end_date.year, month=end_date.month, day=1)
timeslots = (
(datetime_to_timeslot(start_date, level), datetime_to_timeslot(end_date, level)) \
for level in TIMESLOT_LEVEL_NAMES
)
F = ChannelStats.F
removed_count = 0
for start_ts, end_ts in timeslots:
t0 = datetime.now()
res = ChannelStats.objects.coll.remove({
F('time_slot'): {
'$lte': end_ts,
'$gt': start_ts
},
F('channel'): channel.id
})
LOGGER.info(
"purging Q:: channel: %s; collection: ChannelStats; func: %s; timedelta: %s"
% (channel.title, inspect.stack()[0][3], datetime.now() - t0))
removed_count += res['n']
return removed_count
def mark_items_to_keep(channel_or_tag,
time_slot,
rank=None,
parent=None,
call_depth=0):
'''
Tail-recursive algorithm to mark top topics. Sets them to a RESET value. If rank
is specified, use it, and divide by 2 each time, otherwise use the call_depth.
'''
# Sort out the topic limit
if rank == None:
topic_limit = get_var('PURGING_POLICY')[str(call_depth)]
else:
topic_limit = rank
rank = max(rank / 2, 1)
topics = fetch_child_topics(channel_or_tag, time_slot, topic_limit, parent)
doc_ids = [x for x in get_document_ids(channel_or_tag, time_slot, topics)]
# Mark the children to keep them
update = mark_items_to_keep_query(doc_ids)
marked = update['n']
if call_depth <= 1:
for topic in topics:
marked += mark_items_to_keep(channel_or_tag,
time_slot,
rank=rank,
parent=topic,
call_depth=call_depth + 1)
return marked
def __init__(self):
self._git_commit = None
self.load()
if get_var('APP_MODE') == 'prod':
self.validate()
self.validate_kafka()
def test_migration(self):
''' Create a few channels and migrate channel filters for them'''
stored_filter_cls = get_var('CHANNEL_FILTER_CLS')
settings.CHANNEL_FILTER_CLS = 'DbChannelFilter'
channels = []
channel_filters = []
for i in range(1):
self.channel = TwitterChannel.objects.create_by_user(
self.user,
title='C%d' % i,
type='twitter',
intention_types=SA_TYPES)
self.channel.channel_filter.handle_accept(
self._create_post("I need a laptop"))
self.channel.channel_filter.handle_reject(
self._create_post("I hate my laptop"))
channels.append(self.channel)
channel_filters.append(self.channel.channel_filter)
for channel in channels:
migrate_channel_filter(channel, 'OnlineChannelFilter')
self.channel = channel
self.assertTrue(
self.channel.channel_filter._predict_fit(
self._create_post("I need a laptop")) > 0.5)
self.assertTrue(
self.channel.channel_filter._predict_fit(
self._create_post("I hate my laptop")) < 0.5)
settings.CHANNEL_FILTER_CLS = stored_filter_cls
def test_resource_uri(self):
new_acc1 = Account.objects.create(name='TestAccount1')
user_email = 'admin2@test_channels.com'
user_pass = '******'
admin = self._create_db_user(email=user_email, password=user_pass, roles=[ADMIN])
new_acc1.add_perm(admin)
TwitterServiceChannel.objects.create_by_user(admin, title='TSC')
FacebookServiceChannel.objects.create_by_user(admin, title='FSC')
ServiceChannel.objects.create_by_user(admin, title='SC')
token = self.get_token(user_email, user_pass)
post_data = dict(token=token)
data = json.dumps(post_data)
resp = self.client.get('/api/v2.0/channels',
data=data,
content_type='application/json',
base_url='https://localhost')
resp_data = json.loads(resp.data)
self.assertTrue(resp_data['ok'])
channels_list = resp_data['list']
# Only service channel should be returned
self.assertEqual(len(channels_list), 3)
for channel_data in channels_list:
test_uri = channel_data['uri'].replace(get_var('HOST_DOMAIN'), '')
uri_resp = self.client.get(test_uri,
data=data,
content_type='application/json',
base_url='https://localhost')
uri_data = json.loads(uri_resp.data)
uri_channel_data = uri_data['item']
self.assertDictEqual(channel_data, uri_channel_data)
def run(self):
self.ds_client = None
while not self.stopped():
try:
del self.ds_client # to garbage-collect the old client ASAP
self._running = False
if not get_var('ON_TEST'):
self.ds_client = DatasiftClient(ds_login=self.ds_login,
ds_api_key=self.ds_api_key,
bot_instance=self,
sanity_checker=self.checker)
else:
self.ds_client = TestDatasiftClient(bot_instance=self)
self.ds_client.connect()
self._running = True
LOGGER.info('connected to %s', self.ds_client.WEBSOCKET_BASE_URL)
self.checker.set_client(self.ds_client)
self.ds_subscriber.set_client(self.ds_client)
self.ds_client.run() # receives posts from Datasift
except Exception as e:
LOGGER.error(e, exc_info=True)
sleep(5) # wait a bit on any unexpected error
def test_purge_conversations_1(self):
""" Deleting outdated conversation of two posts"""
self._make_setup_for_conversations()
self._create_db_post(channel=self.sc.inbound,
content="I need a foo.",
demand_matchables=True,
user_profile={'screen_name': 'customer'})
self._create_db_post(channel=self.sc.inbound,
content="Does anyone have a foo?",
demand_matchables=True,
user_profile={'screen_name': 'customer'})
self.assertEqual(Conversation.objects.count(), 1)
self.assertEqual(Post.objects.count(), 2)
self.assertEqual(SpeechActMap.objects.count(), 2)
inbound_channel = Channel.objects.get(id=self.sc.inbound)
purge_channel_entities(
inbound_channel,
run_in_prod_mod=True,
now_date=now() +
timedelta(days=get_var("CHANNEL_ENTITIES_KEEP_DAYS") + 1))
self.assertEqual(Conversation.objects.count(), 0)
self.assertEqual(SpeechActMap.objects.count(), 0)
self.assertEqual(Post.objects.count(), 0)
def twitter_handle_id(self):
if get_var('ON_TEST') and not self.twitter_handle_data:
return self.twitter_handle
profile_data = self.get_twitter_profile()
if profile_data:
return profile_data['id_str']
def sf_close_conversation(conversation):
"""
Closes conversation on Salesforce side.
For this we need to patch `Status_in_Social_Optimizr__c` field.
"""
from solariat_bottle.db.account import AccessTokenExpired
if not conversation.service_channel.account.account_type == 'Salesforce':
raise SalesforceException("The account type of this conversation channel is not Salesforce")
access_token = conversation.service_channel.account.access_token
instance_url = conversation.service_channel.account.sf_instance_url
if not instance_url or not access_token:
logger.error("SFDC account is not authorized. Instance url is: %s and access token is %s" % (
str(instance_url), str(access_token)))
headers = {
'X-PrettyPrint' : '1',
'Authorization' : 'Bearer %s' % access_token,
"Content-Type" : "application/json"
}
case_resource = '/services/data/v28.0/sobjects/Case/{}'.format(conversation.external_id)
case_url = instance_url + case_resource
print 'Closing case_resource: ', case_resource
sfdc_prefix = get_var('SFDC_MANAGE_PACKAGE_PREFIX', '')
data = {
sfdc_prefix + "Status_in_Social_Optimizr__c": "Closed"
}
resp = requests.patch(case_url, data=json.dumps(data), headers=headers)
# 204 status code means that request went through, but no body was necessary to return
# this is what Salesforce returns for succesful PATCH request above
if resp.status_code != 204:
if resp.status_code == 401:
# Invalid or expired token, refresh it and try again
conversation.service_channel.account.refresh_access_token()
resp = requests.patch(case_url, data=json.dumps(data), headers=headers)
if resp.status_code == 401:
raise AccessTokenExpired("Invalid or expired token.")
# set external_id to None if Case with this id does not exist
if resp.status_code == 404:
conversation.external_id = None
conversation.save()
raise Exception("Case does not exist. `external_id` is set to `None`.")
if resp.status_code != 204:
error_message = "An error occured while posting to Salesforce. "
error_message += "Response code is: {}".format(resp.status_code)
logger.error(error_message)
raise Exception(error_message)
return True
def clear_matches(user):
data = _get_request_data()
channel_id = data['channel_id']
if get_var('ON_TEST'):
reset_channel_data.sync(channel_id)
else:
reset_channel_data.ignore(channel_id)
return jsonify(ok=True, message="Task for channel reset was started.")
def share_post(self, post, user, dry_run=False):
# self.sync_contacts(post.user_profile)
from solariat_bottle.tasks.twitter import tw_share_post
post_content = post.plaintext_content
status_id = post.native_id
if dry_run is False and not get_var('ON_TEST') and get_var(
'APP_MODE') == 'prod':
tw_share_post.ignore(self,
status_id=status_id,
screen_name=post.user_profile.user_name)
else:
create_outbound_post(user, self, "RT: %s" % post_content, post)
LOGGER.debug("Retweet '%s' using %s", post_content, self)
def test_discard_junk(self):
LONG_TIME_AGO = now() - timedelta(
days=(get_var('HOT_TOPICS_MONTH_STATS_KEEP_MONTHS') + 1) * 30)
ChannelHotTopics.objects.coll.remove()
self._make_laptops_and_icecream(LONG_TIME_AGO)
self.assertEqual(ChannelHotTopics.objects.count(), 10)
print_db_records()
discard_outdated_topics_for_month_level(self.channel)
self.assertEqual(ChannelHotTopics.objects.count(), 5)
def setUp(self):
RestCase.setUp(self)
self.api_token = get_var('ANGEL_API_TOKEN')
# superuser is needed for Angel user creation
self.su = self._create_db_user(email="*****@*****.**",
roles=[STAFF])
self.su.is_superuser = True
self.su.save()
def unsubscribe_from_tag_alerts(email_tag_id):
s = URLSafeSerializer(get_var('UNSUBSCRIBE_KEY'), get_var('UNSUBSCRIBE_SALT'))
try:
user_email, tag_id = s.loads(email_tag_id)
except BadSignature:
abort(404)
t = SmartTagChannel.objects.get(id=tag_id)
t.alert_emails = list(set(t.alert_emails).difference(set([user_email])))
# deactivate alert if no one is subscribing to alert email
if not t.alert_emails:
t.alert_is_active = False
t.save()
return render_template("/unsubscribe_tag.html",
tag_title=t.title,
tag_id=str(t.id))
def _make_posts_and_stats(self):
date_now = now()
date_old = now() - timedelta(days=get_var('CHANNEL_STATS_KEEP_DAYS') +
1)
content_list = [(date_old, "old post")]
self._create_posts(content_list)
content_list = [(date_now, "new post")]
self._create_posts(content_list)
def _handle_create_parameters(self, _id, kw):
'''
This function handles pre-processing of parameters so that events are loaded
with correct parameter mapping for native ids and links to pervious events.
Sets:
* _user_profile from user_profile or actor_id
* _created - will be used ot set with current time
* in_reply_to_native_id
* parent_event
* id creation
kw already must contain:
* actor_id - usually created in tasks.normalize_post_params
* is_inbound - to determine and bind event exact to customer
* event_type - display_name or instance of BaseEventType
'''
if 'user_profile' in kw:
if not get_var('ON_TEST', False): # _obtain_user_profile creates anonymous UserProfile
assert isinstance(kw['user_profile'], self.doc_class.PROFILE_CLASS), \
"(%s) %s is not instance of %s\ndata:\n%s" % \
(type(kw['user_profile']),
kw['user_profile'],
self.doc_class.PROFILE_CLASS,
str(kw['user_profile'].data))
assert _id, 'Id must be generated earlier, with corresponding dynamic ustomer profile'
kw['_user_profile'] = kw.pop('user_profile').id
if 'channels' in kw:
kw['channels'] = [(c.id if isinstance(c, Channel) else c) for c in kw['channels']]
# actor_id must be set on kw earlier, we bind event to actor_id
actor_id = kw['actor_id']
if not kw.get('_created'):
kw['_created'] = now()
in_reply_to_native_id = kw.pop('in_reply_to_native_id', None)
parent_event = kw.pop('parent_event', None)
event_type = kw.get('event_type')
if isinstance(event_type, BaseEventType):
kw['event_type'] = event_type.display_name
# TODO: pass actor_num instead actor_id
kw['id'] = long(
_id or self.doc_class.gen_id(
kw['is_inbound'],
actor_id, kw['_created'],
in_reply_to_native_id,
parent_event
)
)
# Make sure we strip out reference to 'native_id' key. Use it if it is there. If not
# then try for _native_id, or as a last resort, set the native_id to be based directly
# in the id.
kw['_native_id'] = kw.pop('native_id', kw.get('_native_id', str(kw['id'])))
def create_instance(self, channel, cls_name=None):
'''
A Factory class based on config settings.
'''
class_name = cls_name if cls_name != None else get_var(
'CHANNEL_FILTER_CLS', 'OnlineChannelFilter')
channel_filter = get_channel_filter_class(class_name).objects.create(
channel=channel)
self._set_cache(channel_filter.id, channel_filter)
return channel_filter
def get_auth_pool():
"""Fetching AUTH_POOL from settings,
AUTH_POOL is a list of tuples [(
TWITTER_CONSUMER_KEY, TWITTER_CONSUMER_SECRET,
TWITTER_ACCESS_TOKEN, TWITTER_ACCESS_TOKEN_SECRET), (...)]
"""
from solariat_bottle.utils.config import sync_with_keyserver
from solariat_bottle import settings
sync_with_keyserver()
return settings.get_var('AUTH_POOL', [AuthPool.get_auth_tuple()])
def get_current_version():
doc = self.objects.coll.find_one({"_id": find_query["_id"]})
if doc:
version = doc[_v]
else:
version = 1
if get_var('_TEST_TRANSACTION_FAILURE'):
time.sleep(1)
return version
def _get_top_topics(channel, time_slot, topic_set, depth, parent=None):
""" Return the top topics for a given channel and timeslot. Update
a topic_set parameter as you go 'down' from unigrams to bigrams and trigrams """
from solariat_bottle.utils.purging import fetch_child_topics
topic_limit = get_var('PURGING_POLICY')[str(depth)]
topics = fetch_child_topics(channel, time_slot, topic_limit, parent)
topic_set.update(topics)
if depth <= 1:
for topic in topics:
_get_top_topics(channel, time_slot, topic_set, depth + 1, topic)
def list_subscriptions():
"""
List all subscriptions we have on datasift for this given host domain.
:returns A list with all the datasift subscription in the form of a dictionaries.
Example subscription: {u'created_at': 1407414962,
u'end': None,
u'hash': u'2bb5c8a645c444c2351c',
u'hash_type': u'historic',
u'id': u'8d3047e3cc7090887f7db3e2b470318d',
u'last_request': None,
u'last_success': None,
u'lost_data': False,
u'name': u'data_recovery',
u'output_params': {u'delivery_frequency': 30,
u'format': u'json',
u'method': u'post',
u'url': u'http://50.56.112.111:3031/datasift',
u'verify_ssl': u'False'},
u'output_type': u'http',
u'remaining_bytes': None,
u'start': 1407414962,
u'status': u'active',
u'user_id': 2490}
"""
params = {
'username': get_var('DATASIFT_USERNAME'),
'api_key': get_var('DATASIFT_API_KEY')
}
resp = requests.get(
get_var('DATASIFT_BASE_URL') + "/v1/push/get",
urllib.urlencode(params))
try:
data = resp.json()
except ValueError:
raise Exception(resp)
valid_subscriptions = []
for entry in data.get('subscriptions', []):
if get_var('HOST_DOMAIN') in entry.get('output_params', {}).get('url'):
valid_subscriptions.append(entry)
return valid_subscriptions
def test_outdated_trends3(self):
"""
all existing hour stats should be removed, cause it's too old
"""
date_now = now()
date_old = now() - relativedelta(
days=get_var('TOPIC_TRENDS_HOUR_STATS_KEEP_DAYS'), hours=1)
LOGGER.info(
"11111111, %s, %s, %s" %
(date_now, date_old, get_var('TOPIC_TRENDS_HOUR_STATS_KEEP_DAYS')))
self._make_laptops_and_icecream(_created=date_old)
total_trends = ChannelTopicTrends.objects().count()
hour_trends = total_trends / 2
day_trends = total_trends / 2
stats = purge_stats(self.channel)
self.assertEqual(day_trends, 6)
self.assertEqual(hour_trends, 6)
self.assertEqual(stats['discard_junk_stats']['trends_day_count'], 0)
self.assertEqual(stats['discard_junk_stats']['trends_hour_count'], 6)
def test_matchable_search_limit(self):
for i in range(10):
self._create_db_matchable(
creative='There is some foo in location%i' % i,
intention_topics=['foo'])
post = self._create_post('i need some foo')
self.assertEqual(len(post['matchables']), get_var('MATCHABLE_SEARCH_LIMIT'))
settings.MATCHABLE_SEARCH_LIMIT = 8
post = self._create_post('where i can get some foo?')
self.assertEqual(len(post['matchables']), 8)
def test_purge_conversations_2(self):
""" Deleting outdated conversation using inbound and outbound channels """
self._make_setup_for_conversations()
# Generate inbound
post = self._create_tweet(
user_profile=self.contact,
channels=[self.inbound],
content="I need a foo. Does anyone have a foo?")
# reply to post in outbound channels
self._create_tweet(user_profile=self.support,
channels=[self.outbound],
content="I do",
in_reply_to=post)
self.assertEqual(Conversation.objects.count(), 1)
self.assertEqual(Post.objects.count(), 2)
self.assertEqual(SpeechActMap.objects.count(), 3)
inbound_channel = Channel.objects.get(id=self.sc.inbound)
purge_channel_entities(
inbound_channel,
run_in_prod_mod=True,
now_date=now() +
timedelta(days=get_var("CHANNEL_ENTITIES_KEEP_DAYS") + 1))
self.assertEqual(Conversation.objects.count(), 0)
self.assertEqual(Post.objects.count(), 0)
self.assertEqual(SpeechActMap.objects.count(), 1)
outbound_channel = Channel.objects.get(id=self.sc.outbound)
purge_channel_entities(
outbound_channel,
run_in_prod_mod=True,
now_date=now() +
timedelta(days=get_var("CHANNEL_ENTITIES_KEEP_DAYS") + 1))
self.assertEqual(Conversation.objects.count(), 0)
self.assertEqual(Post.objects.count(), 0)
self.assertEqual(SpeechActMap.objects.count(), 0)
