def import_data(cls, user, channel, data_loader): from solariat_bottle.db.post.utils import factory_by_user from solariat_bottle.db.events.event_type import StaticEventType static_event_types_map = {et.id: et for et in StaticEventType.objects.find()} stats = {'total': 0, 'success': 0} for idx, raw_data in enumerate(data_loader.load_data()): if isinstance(raw_data, tuple): # JsonDataLoader returns (event_type, data) tuples event_type_id, raw_data = raw_data # check we can import this event to this channel event_type = static_event_types_map.get(event_type_id) # TODO: wrong! if event_type and event_type.platform != channel.platform.lower(): continue stats['total'] += 1 raw_data.pop('channel', None) raw_data.pop('channels', None) raw_data['channel'] = channel try: factory_by_user(user, **raw_data) except: LOGGER.warning("Cannot import post #%d %s", idx, raw_data, exc_info=True) else: stats['success'] += 1 return stats
def fb_get_comments_for(channel, target, user, since): from solariat_bottle.db.post.utils import factory_by_user from solariat_bottle.daemons.facebook.facebook_data_handlers import FBDataHandlerFactory from solariat_bottle.daemons.facebook.facebook_history_scrapper import FacebookHistoryScrapper puller = FacebookHistoryScrapper(channel, user=user) comments = puller.get_comments(target, since=since)['data'] for comment in comments: comment = puller.handle_data_item( data=comment, handler=FBDataHandlerFactory.get_instance( FBDataHandlerFactory.COMMENT), target_id=target) factory_by_user(user, sync=True, **comment)
def _create_event_sequence(self, customer_up, length=5): kw = {'user': self.user} events = [] post = None for i in range(length): status_id = fake_status_id() if 0 == i % 2: channel = self.sc.inbound_channel user_profile = customer_up else: channel = self.sc.outbound_channel user_profile = self.support_up if post is not None: twitter_data = { 'twitter': { 'in_reply_to_status_id': post.extra_fields['twitter']['id'], 'id': status_id } } else: twitter_data = {'twitter': {'id': status_id}} post = factory_by_user(self.user, channels=[str(channel.id)], content=self.CONTENT[i][1], user_profile=user_profile, **twitter_data) events.append(post) return events
def _create_post(self, user, *args, **kwargs): base_id = '696931697082350_' generated_id = base_id + str(int(time.time() * 1000)) content = kwargs['content'] channel = kwargs['channel'] # Fill in custom content R_POST_OBJECTDATA['content'] = content R_POST_OBJECTDATA['facebook']['_wrapped_data']['message'] = content R_POST_OBJECTDATA['facebook']['_wrapped_data']['id'] = generated_id R_POST_OBJECTDATA['facebook']['facebook_post_id'] = generated_id R_POST_OBJECTDATA['channel'] = channel factory_by_user(user, **R_POST_OBJECTDATA) return dict(ok=True)
def test_fb_dump_concurrency_serial(self): FacebookPost.objects.remove(id__ne=1) Conversation.objects.remove(id__ne=1) fb_srv = FacebookServiceChannel.objects.create_by_user(self.user, title='FSC') channel_id = str(fb_srv.id) self.assertEquals(FacebookPost.objects.count(), 0) self.assertEquals(Conversation.objects.count(), 0) data_dump = open(os.path.join(os.path.dirname(__file__), 'data', 'fb_dump.json')) n_posts = 0 root_ids = [] for line in data_dump.readlines(): n_posts += 1 post_data = json.loads(line) post_data['channel'] = channel_id post = factory_by_user(self.user, **post_data) root_ids.append(post.get_conversation_root_id()) self.assertEquals(FacebookPost.objects.count(), 25) self.assertEquals(Conversation.objects.count(), 1) conv = Conversation.objects.find_one() self.assertEqual(FacebookPost.objects.count(), len(conv.posts)) self.assertEquals(FacebookPost.objects.count(), n_posts) self.assertTrue(conv.has_initial_root_post())
def _preprocess_params(self, user, **kwargs): if 'channel_id' not in kwargs: raise exc.InvalidParameterConfiguration( "Required parameter 'Perform' missing.") if 'content' not in kwargs and 'post_id' not in kwargs: raise exc.InvalidParameterConfiguration( "Either parameter 'content' or 'post_id' required in POST fields." ) try: self.channel = Channel.objects.get(kwargs['channel_id']) except Channel.DoesNotExist: raise exc.ResourceDoesNotExist("No channel found with id=%s" % kwargs['channel_id']) if kwargs.get('post_id', False): post = Post.objects.get(kwargs['post_id']) elif kwargs.get('content', False): target_channel = self.channel.id if not isinstance(self.channel, SmartTagChannel) else \ self.channel.parent_channel post = factory_by_user(user, channels=[target_channel], content=kwargs['content']) else: raise Exception("Either post_id or content should be passed") return post
def make_post(system_user, channel, content, event_name="system_logging", account="Solariat", user="******"): ''' Take a post dictionary and make a post for logging''' from solariat_bottle.db.post.utils import factory_by_user from solariat_nlp import sa_labels # Topics are created to allow easier analysis topics = [ "%s %s %s" % (event_name, account, user), "%s %s" % (user, event_name), "%s %s" % (account, event_name), "%s %s %s" % (account, user, event_name), ] topics = [t.lower() for t in topics] speech_act = dict( content = content, intention_type = "Junk", intention_type_id = sa_labels.SATYPE_TITLE_TO_ID_MAP["Junk"], intention_type_conf = 1.0, intention_topics = topics, intention_topic_conf = 1.0 ) post_content = "%s. %s%s%s" % (content, account, user, event_name) post = factory_by_user(system_user, channel=channel, content=post_content, speech_acts=[speech_act], sync=True) return post
def search_match(user): data = _get_request_data() BenchmarkQuestion.objects.create(received_data=data) # For internal book-keeping post_content = data['post_content'] channel_id = data['channel_id'] post = factory_by_user(user, content=post_content, channel=channel_id) return jsonify(ok=True, items=sorted([{'creative': _d['creative'], 'relevance': _d['relevance'], 'id': _d['id']} for _d in post._get_matchable_dicts()[0]], key=lambda x: -x['relevance']))
def _create_doc(self, user, format_docs=True, *args, **kwargs): """ Create a new document in the collection """ CustomerProfile = user.account.get_customer_profile_class() AgentProfile = user.account.get_agent_profile_class() from solariat_bottle.utils.views import parse_bool return_response = parse_bool(kwargs.pop('_return_response', format_docs)) event_type = kwargs.pop('type', None) model_klass = self.EVENT_MODEL_MAP.get(event_type, Event) assert kwargs.get('native_id') or kwargs.get('actor_id'), (kwargs.get('native_id'), kwargs.get('actor_id')) native_id = kwargs.pop('native_id', None) if kwargs.get('journey_stages'): kwargs['journey_stages'] = self._upsert_journey_stage_pair(user, kwargs['journey_stages']) if native_id: assert native_id, kwargs profile_cls = model_klass.PROFILE_CLASS try: platform_profile = profile_cls.objects.get(native_id=native_id) except profile_cls.DoesNotExist: platform_profile = profile_cls.objects.create(native_id=native_id) platform_profile.save() AccountProfileCls = kwargs['is_inbound'] and CustomerProfile or AgentProfile channel = Channel.objects.get(kwargs['channels'][0]) try: actor_profile = AccountProfileCls.objects.get( account_id=channel.account.id, linked_profile_ids__in=[str(platform_profile.id)] ) except AccountProfileCls.DoesNotExist: actor_profile = AccountProfileCls.objects.create( account_id=channel.account.id, linked_profile_ids=[str(platform_profile.id)] ) kwargs['actor_id'] = str(actor_profile.id) elif kwargs.get('actor_id'): if kwargs['is_inbound']: kwargs['actor_id'] = CustomerProfile.objects.get(kwargs.get('actor_id')).id else: kwargs['actor_id'] = AgentProfile.objects.get(kwargs.get('actor_id')).id else: raise Exception('either "native_id" or "actor_id" should be provided') if event_type in ('chat', 'voice', 'twitter', 'facebook'): doc = factory_by_user(user, **kwargs) else: doc = model_klass.objects.create_by_user(user, safe_create=True, **kwargs) if return_response and format_docs: return self._format_single_doc(doc) if not return_response: return {"ok": True} return doc
def _create_comment(self, user, *args, **kwargs): base_id = '696931697082350_' generated_id = base_id + str(int(time.time() * 1000)) content = kwargs['content'] channel = kwargs['channel'] parent_id = kwargs.get('parent', None) # Fill in custom content R_POST_OBJECTDATA['content'] = content R_POST_OBJECTDATA['facebook']['_wrapped_data']['message'] = content R_POST_OBJECTDATA['facebook']['_wrapped_data']['id'] = generated_id R_POST_OBJECTDATA['facebook']['facebook_post_id'] = generated_id R_POST_OBJECTDATA['channel'] = channel if parent_id: R_POST_OBJECTDATA['facebook']['in_reply_to_status_id'] = parent_id factory_by_user(user, **R_POST_OBJECTDATA) return dict(ok=True)
def fb_get_post_comments(channel, user, target, since, until): from solariat_bottle.db.post.utils import factory_by_user from solariat_bottle.daemons.facebook.facebook_data_handlers import FBDataHandlerFactory from solariat_bottle.daemons.facebook.facebook_history_scrapper import FacebookHistoryScrapper puller = FacebookHistoryScrapper(channel, user) new_posts = puller.get_posts(since=since, until=until, target=target) for post in new_posts['data']: #avoid handling posts with no data useful to parsing if 'message' not in post and 'link' not in post: continue try: post = puller.handle_data_item( data=post, handler=FBDataHandlerFactory.get_instance( FBDataHandlerFactory.POST), target_id=target) except Exception, ex: logger.error(ex) continue comments = post.pop('comments', []) factory_by_user(user, sync=True, **post) for comment in comments: post_id = post['id'] try: comment = puller.handle_data_item( data=comment, handler=FBDataHandlerFactory.get_instance( FBDataHandlerFactory.COMMENT), target_id=post_id) except Exception, ex: logger.error(ex) continue factory_by_user(user, sync=True, **comment)
def fb_get_private_messages(channel, page_id, user, since, until): from solariat_bottle.db.post.utils import factory_by_user from solariat_bottle.daemons.facebook.facebook_data_handlers import FBDataHandlerFactory from solariat_bottle.daemons.facebook.facebook_history_scrapper import FacebookHistoryScrapper puller = FacebookHistoryScrapper(channel, user=user) message_threads = puller.get_page_private_messages(page_id, since, until)['data'] for thread in message_threads: if 'messages' in thread and thread['messages']['data']: data = thread['messages'][ 'data'][::-1] #get array of messages in reversed order root_message_id = data[0]['id'] if thread['id'] in channel.tracked_fb_message_threads_ids: check = lambda msg, since: utc( parser.parse(msg['created_time'])) > utc(since) messages_to_handle = [msg for msg in data if check(msg, since)] else: channel.tracked_fb_message_threads_ids.append(thread['id']) channel.save() messages_to_handle = data conversation_id = thread['id'] for msg in messages_to_handle: if msg['id'] != root_message_id: msg['root_post'] = root_message_id msg['conversation_id'] = conversation_id msg['page_id'] = page_id msg = puller.handle_data_item( msg, FBDataHandlerFactory.get_instance(FBDataHandlerFactory.PM), thread['id']) factory_by_user(user, sync=True, **msg)
def test_workflow(self): """ Testing Predictive Matching coupled with Conversation State Machine """ # creating three posts customer_post = factory_by_user( self.user, channels=[str(self.sc.inbound)], content='Hi, I need a laptop battery', user_profile=self.chat_profile.customer_profile, extra_fields={'chat': { 'session_id': 'test_sessin_id' }}) brand_post = factory_by_user( self.user, channels=[str(self.sc.outbound)], content='Hey, we will ship new battery to you asap', actor_id=self.agent_profile.id, is_inbound=False, extra_fields={ 'chat': { 'session_id': 'test_sessin_id', 'in_reply_to_status_id': customer_post.id } }) customer_post2 = factory_by_user( self.user, channels=[str(self.sc.inbound)], content="Thank you! That's very nice!", user_profile=self.customer, extra_fields={ 'chat': { 'session_id': 'test_sessin_id', 'in_reply_to_status_id': brand_post.id } }) self.assertEqual(Conversation.objects().count(), 1)
def _create_doc(self, user, *args, **kwargs): def parse_bool(val): return val in ('', None, True, 'true', 'True', 'y', 'yes') add_to_queue = parse_bool(kwargs.pop('add_to_queue', True)) format_docs = parse_bool(kwargs.pop('format_docs', True)) return_response = parse_bool(kwargs.pop('return_response', True)) if {'serialized_to_json', 'post_object_data'} <= set(kwargs.keys()) \ and parse_bool(kwargs['serialized_to_json']): post_kwargs = json.loads(kwargs['post_object_data']) else: post_kwargs = kwargs post_kwargs.setdefault('add_to_queue', add_to_queue) post = factory_by_user(user, **post_kwargs) if not return_response: return {'ok': True} if format_docs: return self._format_single_doc(post) return post
def create_post(self, **data): from solariat_bottle.db.post.utils import factory_by_user return factory_by_user(self.user, **data)
def test_event(self): # testing PostEvent creation customer_post = factory_by_user(self.user, channels=[self.sc.inbound], content='Hi, I need a laptop battery', user_profile=self.customer_up)
def create_post(self, **data): from solariat_bottle.db.post.utils import factory_by_user from solariat_bottle.db.user import User user = User.objects.get(email=self.user) return factory_by_user(user, **data)