def tw_unfollow(channel, user_profile, silent_ex=False): """ Task for unfollowing a twitter screen name. :param channel: Used to grab the twitter credentials we are using for the unfollow action. :param screen_name: The twitter screen name we are going to unfollow :param silent_ex: Optional, if true any exceptions will just be silently ignored """ from solariat_bottle.settings import get_var, LOGGER from solariat_bottle.db.user_profiles.user_profile import UserProfile, get_brand_profile_id, get_native_user_id_from_channel result = {} if not get_var('ON_TEST') and get_var('APP_MODE') == 'prod': from solariat_bottle.utils.oauth import get_twitter_oauth_credentials (consumer_key, consumer_secret, _, access_token_key, access_token_secret) = get_twitter_oauth_credentials(channel) try: result = tw_destroy_friendship( consumer_key=consumer_key, consumer_secret=consumer_secret, access_token_key=access_token_key, access_token_secret=access_token_secret, screen_name=user_profile.screen_name) except Exception, ex: LOGGER.error("tasks.tw_unfollow: " + str(ex)) if silent_ex: result = dict(error=str(ex)) else: raise
def generate_shortened_url(long_url, access_token=False): """Generate bitly url, see code.google.com/p/bitly-api/wiki/ApiDocumentation#/v3/shorten """ params = dict(longUrl=long_url) if access_token: params['access_token'] = access_token else: params['login'] = get_var('BITLY_LOGIN') params['apiKey'] = get_var('BITLY_KEY') if get_var('BITLY_DOMAIN'): params['domain'] = get_var('BITLY_DOMAIN') url = "%sshorten?%s" % (get_var('BITLY_BASE_URL'), urllib.urlencode(params)) resp = requests.get(url) if not resp.status_code == 200: raise RuntimeError("Could not get shorten url, %s" % resp.headers) result = resp.json() try: return result['data']['url'] except (KeyError, TypeError): raise RuntimeError("Could not get shorten %s, %s" % (long_url, result))
def __init__(self): self.INDEX_NAME = get_var( 'CHANNEL_FILTER_ITEM_INDEX_NAME') or 'channelfilteritems' self.DOCUMENT_NAME = get_var( 'CHANNEL_FILTER_ITEM_DOCUMENT_NAME') or 'channelfilteritem' ElasticCollection.__init__(self)
def tearDown(self): if get_var('TEST_STAT_RESET', False): to_date = now() from_date = to_date - timedelta(days=90) exceptions = get_var('EXCEPTED_RESET_CASES', []) test_name = self.__class__.__name__ + '.' + self._testMethodName if test_name not in exceptions: try: for account in Account.objects(): do_it(test_mode=True, raise_on_diffs=True, account_name=account.name, to_date=to_date, from_date=from_date) except Exception, ex: self.fail(ex)
def clear_subscriptions(): """ Clear all subscriptions we have on datasift for this given host domain. """ params = { 'username': get_var('DATASIFT_USERNAME'), 'api_key': get_var('DATASIFT_API_KEY') } existing_subs = list_subscriptions() for subscription in existing_subs: params['id'] = subscription['id'] requests.get( get_var("DATASIFT_BASE_URL") + "/v1/push/delete", urllib.urlencode(params))
def send_notification_for_team(subject, body=None): suffix = "| host: '%s' at %s'" % (gethostname(), datetime.now()) subject = subject + suffix if body is None: body = subject with app.app_context(): msg = Message(subject=subject, sender=get_var("MAIL_DEFAULT_SENDER"), recipients=get_var("SCRIPTS_EMAIL_LIST"), body=body) if get_var('APP_MODE') == 'prod': mail.send(msg) else: LOGGER.info(msg.subject) LOGGER.info(msg.body)
def purge_channel_stats(channel): days = get_var('CHANNEL_STATS_KEEP_DAYS') start_date = datetime(year=2012, month=1, day=1) end_date = now() - timedelta(days=days) # end_date = datetime(year=end_date.year, month=end_date.month, day=1) timeslots = ( (datetime_to_timeslot(start_date, level), datetime_to_timeslot(end_date, level)) \ for level in TIMESLOT_LEVEL_NAMES ) F = ChannelStats.F removed_count = 0 for start_ts, end_ts in timeslots: t0 = datetime.now() res = ChannelStats.objects.coll.remove({ F('time_slot'): { '$lte': end_ts, '$gt': start_ts }, F('channel'): channel.id }) LOGGER.info( "purging Q:: channel: %s; collection: ChannelStats; func: %s; timedelta: %s" % (channel.title, inspect.stack()[0][3], datetime.now() - t0)) removed_count += res['n'] return removed_count
def mark_items_to_keep(channel_or_tag, time_slot, rank=None, parent=None, call_depth=0): ''' Tail-recursive algorithm to mark top topics. Sets them to a RESET value. If rank is specified, use it, and divide by 2 each time, otherwise use the call_depth. ''' # Sort out the topic limit if rank == None: topic_limit = get_var('PURGING_POLICY')[str(call_depth)] else: topic_limit = rank rank = max(rank / 2, 1) topics = fetch_child_topics(channel_or_tag, time_slot, topic_limit, parent) doc_ids = [x for x in get_document_ids(channel_or_tag, time_slot, topics)] # Mark the children to keep them update = mark_items_to_keep_query(doc_ids) marked = update['n'] if call_depth <= 1: for topic in topics: marked += mark_items_to_keep(channel_or_tag, time_slot, rank=rank, parent=topic, call_depth=call_depth + 1) return marked
def __init__(self): self._git_commit = None self.load() if get_var('APP_MODE') == 'prod': self.validate() self.validate_kafka()
def test_migration(self): ''' Create a few channels and migrate channel filters for them''' stored_filter_cls = get_var('CHANNEL_FILTER_CLS') settings.CHANNEL_FILTER_CLS = 'DbChannelFilter' channels = [] channel_filters = [] for i in range(1): self.channel = TwitterChannel.objects.create_by_user( self.user, title='C%d' % i, type='twitter', intention_types=SA_TYPES) self.channel.channel_filter.handle_accept( self._create_post("I need a laptop")) self.channel.channel_filter.handle_reject( self._create_post("I hate my laptop")) channels.append(self.channel) channel_filters.append(self.channel.channel_filter) for channel in channels: migrate_channel_filter(channel, 'OnlineChannelFilter') self.channel = channel self.assertTrue( self.channel.channel_filter._predict_fit( self._create_post("I need a laptop")) > 0.5) self.assertTrue( self.channel.channel_filter._predict_fit( self._create_post("I hate my laptop")) < 0.5) settings.CHANNEL_FILTER_CLS = stored_filter_cls
def test_resource_uri(self): new_acc1 = Account.objects.create(name='TestAccount1') user_email = 'admin2@test_channels.com' user_pass = '******' admin = self._create_db_user(email=user_email, password=user_pass, roles=[ADMIN]) new_acc1.add_perm(admin) TwitterServiceChannel.objects.create_by_user(admin, title='TSC') FacebookServiceChannel.objects.create_by_user(admin, title='FSC') ServiceChannel.objects.create_by_user(admin, title='SC') token = self.get_token(user_email, user_pass) post_data = dict(token=token) data = json.dumps(post_data) resp = self.client.get('/api/v2.0/channels', data=data, content_type='application/json', base_url='https://localhost') resp_data = json.loads(resp.data) self.assertTrue(resp_data['ok']) channels_list = resp_data['list'] # Only service channel should be returned self.assertEqual(len(channels_list), 3) for channel_data in channels_list: test_uri = channel_data['uri'].replace(get_var('HOST_DOMAIN'), '') uri_resp = self.client.get(test_uri, data=data, content_type='application/json', base_url='https://localhost') uri_data = json.loads(uri_resp.data) uri_channel_data = uri_data['item'] self.assertDictEqual(channel_data, uri_channel_data)
def run(self): self.ds_client = None while not self.stopped(): try: del self.ds_client # to garbage-collect the old client ASAP self._running = False if not get_var('ON_TEST'): self.ds_client = DatasiftClient(ds_login=self.ds_login, ds_api_key=self.ds_api_key, bot_instance=self, sanity_checker=self.checker) else: self.ds_client = TestDatasiftClient(bot_instance=self) self.ds_client.connect() self._running = True LOGGER.info('connected to %s', self.ds_client.WEBSOCKET_BASE_URL) self.checker.set_client(self.ds_client) self.ds_subscriber.set_client(self.ds_client) self.ds_client.run() # receives posts from Datasift except Exception as e: LOGGER.error(e, exc_info=True) sleep(5) # wait a bit on any unexpected error
def test_purge_conversations_1(self): """ Deleting outdated conversation of two posts""" self._make_setup_for_conversations() self._create_db_post(channel=self.sc.inbound, content="I need a foo.", demand_matchables=True, user_profile={'screen_name': 'customer'}) self._create_db_post(channel=self.sc.inbound, content="Does anyone have a foo?", demand_matchables=True, user_profile={'screen_name': 'customer'}) self.assertEqual(Conversation.objects.count(), 1) self.assertEqual(Post.objects.count(), 2) self.assertEqual(SpeechActMap.objects.count(), 2) inbound_channel = Channel.objects.get(id=self.sc.inbound) purge_channel_entities( inbound_channel, run_in_prod_mod=True, now_date=now() + timedelta(days=get_var("CHANNEL_ENTITIES_KEEP_DAYS") + 1)) self.assertEqual(Conversation.objects.count(), 0) self.assertEqual(SpeechActMap.objects.count(), 0) self.assertEqual(Post.objects.count(), 0)
def twitter_handle_id(self): if get_var('ON_TEST') and not self.twitter_handle_data: return self.twitter_handle profile_data = self.get_twitter_profile() if profile_data: return profile_data['id_str']
def sf_close_conversation(conversation): """ Closes conversation on Salesforce side. For this we need to patch `Status_in_Social_Optimizr__c` field. """ from solariat_bottle.db.account import AccessTokenExpired if not conversation.service_channel.account.account_type == 'Salesforce': raise SalesforceException("The account type of this conversation channel is not Salesforce") access_token = conversation.service_channel.account.access_token instance_url = conversation.service_channel.account.sf_instance_url if not instance_url or not access_token: logger.error("SFDC account is not authorized. Instance url is: %s and access token is %s" % ( str(instance_url), str(access_token))) headers = { 'X-PrettyPrint' : '1', 'Authorization' : 'Bearer %s' % access_token, "Content-Type" : "application/json" } case_resource = '/services/data/v28.0/sobjects/Case/{}'.format(conversation.external_id) case_url = instance_url + case_resource print 'Closing case_resource: ', case_resource sfdc_prefix = get_var('SFDC_MANAGE_PACKAGE_PREFIX', '') data = { sfdc_prefix + "Status_in_Social_Optimizr__c": "Closed" } resp = requests.patch(case_url, data=json.dumps(data), headers=headers) # 204 status code means that request went through, but no body was necessary to return # this is what Salesforce returns for succesful PATCH request above if resp.status_code != 204: if resp.status_code == 401: # Invalid or expired token, refresh it and try again conversation.service_channel.account.refresh_access_token() resp = requests.patch(case_url, data=json.dumps(data), headers=headers) if resp.status_code == 401: raise AccessTokenExpired("Invalid or expired token.") # set external_id to None if Case with this id does not exist if resp.status_code == 404: conversation.external_id = None conversation.save() raise Exception("Case does not exist. `external_id` is set to `None`.") if resp.status_code != 204: error_message = "An error occured while posting to Salesforce. " error_message += "Response code is: {}".format(resp.status_code) logger.error(error_message) raise Exception(error_message) return True
def clear_matches(user): data = _get_request_data() channel_id = data['channel_id'] if get_var('ON_TEST'): reset_channel_data.sync(channel_id) else: reset_channel_data.ignore(channel_id) return jsonify(ok=True, message="Task for channel reset was started.")
def share_post(self, post, user, dry_run=False): # self.sync_contacts(post.user_profile) from solariat_bottle.tasks.twitter import tw_share_post post_content = post.plaintext_content status_id = post.native_id if dry_run is False and not get_var('ON_TEST') and get_var( 'APP_MODE') == 'prod': tw_share_post.ignore(self, status_id=status_id, screen_name=post.user_profile.user_name) else: create_outbound_post(user, self, "RT: %s" % post_content, post) LOGGER.debug("Retweet '%s' using %s", post_content, self)
def test_discard_junk(self): LONG_TIME_AGO = now() - timedelta( days=(get_var('HOT_TOPICS_MONTH_STATS_KEEP_MONTHS') + 1) * 30) ChannelHotTopics.objects.coll.remove() self._make_laptops_and_icecream(LONG_TIME_AGO) self.assertEqual(ChannelHotTopics.objects.count(), 10) print_db_records() discard_outdated_topics_for_month_level(self.channel) self.assertEqual(ChannelHotTopics.objects.count(), 5)
def setUp(self): RestCase.setUp(self) self.api_token = get_var('ANGEL_API_TOKEN') # superuser is needed for Angel user creation self.su = self._create_db_user(email="*****@*****.**", roles=[STAFF]) self.su.is_superuser = True self.su.save()
def unsubscribe_from_tag_alerts(email_tag_id): s = URLSafeSerializer(get_var('UNSUBSCRIBE_KEY'), get_var('UNSUBSCRIBE_SALT')) try: user_email, tag_id = s.loads(email_tag_id) except BadSignature: abort(404) t = SmartTagChannel.objects.get(id=tag_id) t.alert_emails = list(set(t.alert_emails).difference(set([user_email]))) # deactivate alert if no one is subscribing to alert email if not t.alert_emails: t.alert_is_active = False t.save() return render_template("/unsubscribe_tag.html", tag_title=t.title, tag_id=str(t.id))
def _make_posts_and_stats(self): date_now = now() date_old = now() - timedelta(days=get_var('CHANNEL_STATS_KEEP_DAYS') + 1) content_list = [(date_old, "old post")] self._create_posts(content_list) content_list = [(date_now, "new post")] self._create_posts(content_list)
def _handle_create_parameters(self, _id, kw): ''' This function handles pre-processing of parameters so that events are loaded with correct parameter mapping for native ids and links to pervious events. Sets: * _user_profile from user_profile or actor_id * _created - will be used ot set with current time * in_reply_to_native_id * parent_event * id creation kw already must contain: * actor_id - usually created in tasks.normalize_post_params * is_inbound - to determine and bind event exact to customer * event_type - display_name or instance of BaseEventType ''' if 'user_profile' in kw: if not get_var('ON_TEST', False): # _obtain_user_profile creates anonymous UserProfile assert isinstance(kw['user_profile'], self.doc_class.PROFILE_CLASS), \ "(%s) %s is not instance of %s\ndata:\n%s" % \ (type(kw['user_profile']), kw['user_profile'], self.doc_class.PROFILE_CLASS, str(kw['user_profile'].data)) assert _id, 'Id must be generated earlier, with corresponding dynamic ustomer profile' kw['_user_profile'] = kw.pop('user_profile').id if 'channels' in kw: kw['channels'] = [(c.id if isinstance(c, Channel) else c) for c in kw['channels']] # actor_id must be set on kw earlier, we bind event to actor_id actor_id = kw['actor_id'] if not kw.get('_created'): kw['_created'] = now() in_reply_to_native_id = kw.pop('in_reply_to_native_id', None) parent_event = kw.pop('parent_event', None) event_type = kw.get('event_type') if isinstance(event_type, BaseEventType): kw['event_type'] = event_type.display_name # TODO: pass actor_num instead actor_id kw['id'] = long( _id or self.doc_class.gen_id( kw['is_inbound'], actor_id, kw['_created'], in_reply_to_native_id, parent_event ) ) # Make sure we strip out reference to 'native_id' key. Use it if it is there. If not # then try for _native_id, or as a last resort, set the native_id to be based directly # in the id. kw['_native_id'] = kw.pop('native_id', kw.get('_native_id', str(kw['id'])))
def create_instance(self, channel, cls_name=None): ''' A Factory class based on config settings. ''' class_name = cls_name if cls_name != None else get_var( 'CHANNEL_FILTER_CLS', 'OnlineChannelFilter') channel_filter = get_channel_filter_class(class_name).objects.create( channel=channel) self._set_cache(channel_filter.id, channel_filter) return channel_filter
def get_auth_pool(): """Fetching AUTH_POOL from settings, AUTH_POOL is a list of tuples [( TWITTER_CONSUMER_KEY, TWITTER_CONSUMER_SECRET, TWITTER_ACCESS_TOKEN, TWITTER_ACCESS_TOKEN_SECRET), (...)] """ from solariat_bottle.utils.config import sync_with_keyserver from solariat_bottle import settings sync_with_keyserver() return settings.get_var('AUTH_POOL', [AuthPool.get_auth_tuple()])
def get_current_version(): doc = self.objects.coll.find_one({"_id": find_query["_id"]}) if doc: version = doc[_v] else: version = 1 if get_var('_TEST_TRANSACTION_FAILURE'): time.sleep(1) return version
def _get_top_topics(channel, time_slot, topic_set, depth, parent=None): """ Return the top topics for a given channel and timeslot. Update a topic_set parameter as you go 'down' from unigrams to bigrams and trigrams """ from solariat_bottle.utils.purging import fetch_child_topics topic_limit = get_var('PURGING_POLICY')[str(depth)] topics = fetch_child_topics(channel, time_slot, topic_limit, parent) topic_set.update(topics) if depth <= 1: for topic in topics: _get_top_topics(channel, time_slot, topic_set, depth + 1, topic)
def list_subscriptions(): """ List all subscriptions we have on datasift for this given host domain. :returns A list with all the datasift subscription in the form of a dictionaries. Example subscription: {u'created_at': 1407414962, u'end': None, u'hash': u'2bb5c8a645c444c2351c', u'hash_type': u'historic', u'id': u'8d3047e3cc7090887f7db3e2b470318d', u'last_request': None, u'last_success': None, u'lost_data': False, u'name': u'data_recovery', u'output_params': {u'delivery_frequency': 30, u'format': u'json', u'method': u'post', u'url': u'http://50.56.112.111:3031/datasift', u'verify_ssl': u'False'}, u'output_type': u'http', u'remaining_bytes': None, u'start': 1407414962, u'status': u'active', u'user_id': 2490} """ params = { 'username': get_var('DATASIFT_USERNAME'), 'api_key': get_var('DATASIFT_API_KEY') } resp = requests.get( get_var('DATASIFT_BASE_URL') + "/v1/push/get", urllib.urlencode(params)) try: data = resp.json() except ValueError: raise Exception(resp) valid_subscriptions = [] for entry in data.get('subscriptions', []): if get_var('HOST_DOMAIN') in entry.get('output_params', {}).get('url'): valid_subscriptions.append(entry) return valid_subscriptions
def test_outdated_trends3(self): """ all existing hour stats should be removed, cause it's too old """ date_now = now() date_old = now() - relativedelta( days=get_var('TOPIC_TRENDS_HOUR_STATS_KEEP_DAYS'), hours=1) LOGGER.info( "11111111, %s, %s, %s" % (date_now, date_old, get_var('TOPIC_TRENDS_HOUR_STATS_KEEP_DAYS'))) self._make_laptops_and_icecream(_created=date_old) total_trends = ChannelTopicTrends.objects().count() hour_trends = total_trends / 2 day_trends = total_trends / 2 stats = purge_stats(self.channel) self.assertEqual(day_trends, 6) self.assertEqual(hour_trends, 6) self.assertEqual(stats['discard_junk_stats']['trends_day_count'], 0) self.assertEqual(stats['discard_junk_stats']['trends_hour_count'], 6)
def test_matchable_search_limit(self): for i in range(10): self._create_db_matchable( creative='There is some foo in location%i' % i, intention_topics=['foo']) post = self._create_post('i need some foo') self.assertEqual(len(post['matchables']), get_var('MATCHABLE_SEARCH_LIMIT')) settings.MATCHABLE_SEARCH_LIMIT = 8 post = self._create_post('where i can get some foo?') self.assertEqual(len(post['matchables']), 8)
def test_purge_conversations_2(self): """ Deleting outdated conversation using inbound and outbound channels """ self._make_setup_for_conversations() # Generate inbound post = self._create_tweet( user_profile=self.contact, channels=[self.inbound], content="I need a foo. Does anyone have a foo?") # reply to post in outbound channels self._create_tweet(user_profile=self.support, channels=[self.outbound], content="I do", in_reply_to=post) self.assertEqual(Conversation.objects.count(), 1) self.assertEqual(Post.objects.count(), 2) self.assertEqual(SpeechActMap.objects.count(), 3) inbound_channel = Channel.objects.get(id=self.sc.inbound) purge_channel_entities( inbound_channel, run_in_prod_mod=True, now_date=now() + timedelta(days=get_var("CHANNEL_ENTITIES_KEEP_DAYS") + 1)) self.assertEqual(Conversation.objects.count(), 0) self.assertEqual(Post.objects.count(), 0) self.assertEqual(SpeechActMap.objects.count(), 1) outbound_channel = Channel.objects.get(id=self.sc.outbound) purge_channel_entities( outbound_channel, run_in_prod_mod=True, now_date=now() + timedelta(days=get_var("CHANNEL_ENTITIES_KEEP_DAYS") + 1)) self.assertEqual(Conversation.objects.count(), 0) self.assertEqual(Post.objects.count(), 0) self.assertEqual(SpeechActMap.objects.count(), 0)