def import_data(cls, user, channel, data_loader):
from solariat_bottle.db.post.utils import factory_by_user
from solariat_bottle.db.events.event_type import StaticEventType
static_event_types_map = {et.id: et for et in StaticEventType.objects.find()}
stats = {'total': 0, 'success': 0}
for idx, raw_data in enumerate(data_loader.load_data()):
if isinstance(raw_data, tuple): # JsonDataLoader returns (event_type, data) tuples
event_type_id, raw_data = raw_data
# check we can import this event to this channel
event_type = static_event_types_map.get(event_type_id) # TODO: wrong!
if event_type and event_type.platform != channel.platform.lower():
continue
stats['total'] += 1
raw_data.pop('channel', None)
raw_data.pop('channels', None)
raw_data['channel'] = channel
try:
factory_by_user(user, **raw_data)
except:
LOGGER.warning("Cannot import post #%d %s", idx, raw_data, exc_info=True)
else:
stats['success'] += 1
return stats
def fb_get_comments_for(channel, target, user, since):
from solariat_bottle.db.post.utils import factory_by_user
from solariat_bottle.daemons.facebook.facebook_data_handlers import FBDataHandlerFactory
from solariat_bottle.daemons.facebook.facebook_history_scrapper import FacebookHistoryScrapper
puller = FacebookHistoryScrapper(channel, user=user)
comments = puller.get_comments(target, since=since)['data']
for comment in comments:
comment = puller.handle_data_item(
data=comment,
handler=FBDataHandlerFactory.get_instance(
FBDataHandlerFactory.COMMENT),
target_id=target)
factory_by_user(user, sync=True, **comment)
def _create_event_sequence(self, customer_up, length=5):
kw = {'user': self.user}
events = []
post = None
for i in range(length):
status_id = fake_status_id()
if 0 == i % 2:
channel = self.sc.inbound_channel
user_profile = customer_up
else:
channel = self.sc.outbound_channel
user_profile = self.support_up
if post is not None:
twitter_data = {
'twitter': {
'in_reply_to_status_id':
post.extra_fields['twitter']['id'],
'id': status_id
}
}
else:
twitter_data = {'twitter': {'id': status_id}}
post = factory_by_user(self.user,
channels=[str(channel.id)],
content=self.CONTENT[i][1],
user_profile=user_profile,
**twitter_data)
events.append(post)
return events
def _create_post(self, user, *args, **kwargs):
base_id = '696931697082350_'
generated_id = base_id + str(int(time.time() * 1000))
content = kwargs['content']
channel = kwargs['channel']
# Fill in custom content
R_POST_OBJECTDATA['content'] = content
R_POST_OBJECTDATA['facebook']['_wrapped_data']['message'] = content
R_POST_OBJECTDATA['facebook']['_wrapped_data']['id'] = generated_id
R_POST_OBJECTDATA['facebook']['facebook_post_id'] = generated_id
R_POST_OBJECTDATA['channel'] = channel
factory_by_user(user, **R_POST_OBJECTDATA)
return dict(ok=True)
def test_fb_dump_concurrency_serial(self):
FacebookPost.objects.remove(id__ne=1)
Conversation.objects.remove(id__ne=1)
fb_srv = FacebookServiceChannel.objects.create_by_user(self.user, title='FSC')
channel_id = str(fb_srv.id)
self.assertEquals(FacebookPost.objects.count(), 0)
self.assertEquals(Conversation.objects.count(), 0)
data_dump = open(os.path.join(os.path.dirname(__file__), 'data', 'fb_dump.json'))
n_posts = 0
root_ids = []
for line in data_dump.readlines():
n_posts += 1
post_data = json.loads(line)
post_data['channel'] = channel_id
post = factory_by_user(self.user, **post_data)
root_ids.append(post.get_conversation_root_id())
self.assertEquals(FacebookPost.objects.count(), 25)
self.assertEquals(Conversation.objects.count(), 1)
conv = Conversation.objects.find_one()
self.assertEqual(FacebookPost.objects.count(), len(conv.posts))
self.assertEquals(FacebookPost.objects.count(), n_posts)
self.assertTrue(conv.has_initial_root_post())
def _preprocess_params(self, user, **kwargs):
if 'channel_id' not in kwargs:
raise exc.InvalidParameterConfiguration(
"Required parameter 'Perform' missing.")
if 'content' not in kwargs and 'post_id' not in kwargs:
raise exc.InvalidParameterConfiguration(
"Either parameter 'content' or 'post_id' required in POST fields."
)
try:
self.channel = Channel.objects.get(kwargs['channel_id'])
except Channel.DoesNotExist:
raise exc.ResourceDoesNotExist("No channel found with id=%s" %
kwargs['channel_id'])
if kwargs.get('post_id', False):
post = Post.objects.get(kwargs['post_id'])
elif kwargs.get('content', False):
target_channel = self.channel.id if not isinstance(self.channel, SmartTagChannel) else \
self.channel.parent_channel
post = factory_by_user(user,
channels=[target_channel],
content=kwargs['content'])
else:
raise Exception("Either post_id or content should be passed")
return post
def make_post(system_user, channel, content,
event_name="system_logging",
account="Solariat",
user="******"):
''' Take a post dictionary and make a post for logging'''
from solariat_bottle.db.post.utils import factory_by_user
from solariat_nlp import sa_labels
# Topics are created to allow easier analysis
topics = [
"%s %s %s" % (event_name, account, user),
"%s %s" % (user, event_name),
"%s %s" % (account, event_name),
"%s %s %s" % (account, user, event_name),
]
topics = [t.lower() for t in topics]
speech_act = dict(
content = content,
intention_type = "Junk",
intention_type_id = sa_labels.SATYPE_TITLE_TO_ID_MAP["Junk"],
intention_type_conf = 1.0,
intention_topics = topics,
intention_topic_conf = 1.0
)
post_content = "%s. %s%s%s" % (content, account, user, event_name)
post = factory_by_user(system_user,
channel=channel,
content=post_content,
speech_acts=[speech_act],
sync=True)
return post
def search_match(user):
data = _get_request_data()
BenchmarkQuestion.objects.create(received_data=data) # For internal book-keeping
post_content = data['post_content']
channel_id = data['channel_id']
post = factory_by_user(user, content=post_content, channel=channel_id)
return jsonify(ok=True, items=sorted([{'creative': _d['creative'],
'relevance': _d['relevance'],
'id': _d['id']} for _d in post._get_matchable_dicts()[0]],
key=lambda x: -x['relevance']))
def _create_doc(self, user, format_docs=True, *args, **kwargs):
""" Create a new document in the collection """
CustomerProfile = user.account.get_customer_profile_class()
AgentProfile = user.account.get_agent_profile_class()
from solariat_bottle.utils.views import parse_bool
return_response = parse_bool(kwargs.pop('_return_response', format_docs))
event_type = kwargs.pop('type', None)
model_klass = self.EVENT_MODEL_MAP.get(event_type, Event)
assert kwargs.get('native_id') or kwargs.get('actor_id'), (kwargs.get('native_id'), kwargs.get('actor_id'))
native_id = kwargs.pop('native_id', None)
if kwargs.get('journey_stages'):
kwargs['journey_stages'] = self._upsert_journey_stage_pair(user, kwargs['journey_stages'])
if native_id:
assert native_id, kwargs
profile_cls = model_klass.PROFILE_CLASS
try:
platform_profile = profile_cls.objects.get(native_id=native_id)
except profile_cls.DoesNotExist:
platform_profile = profile_cls.objects.create(native_id=native_id)
platform_profile.save()
AccountProfileCls = kwargs['is_inbound'] and CustomerProfile or AgentProfile
channel = Channel.objects.get(kwargs['channels'][0])
try:
actor_profile = AccountProfileCls.objects.get(
account_id=channel.account.id,
linked_profile_ids__in=[str(platform_profile.id)]
)
except AccountProfileCls.DoesNotExist:
actor_profile = AccountProfileCls.objects.create(
account_id=channel.account.id,
linked_profile_ids=[str(platform_profile.id)]
)
kwargs['actor_id'] = str(actor_profile.id)
elif kwargs.get('actor_id'):
if kwargs['is_inbound']:
kwargs['actor_id'] = CustomerProfile.objects.get(kwargs.get('actor_id')).id
else:
kwargs['actor_id'] = AgentProfile.objects.get(kwargs.get('actor_id')).id
else:
raise Exception('either "native_id" or "actor_id" should be provided')
if event_type in ('chat', 'voice', 'twitter', 'facebook'):
doc = factory_by_user(user, **kwargs)
else:
doc = model_klass.objects.create_by_user(user, safe_create=True, **kwargs)
if return_response and format_docs:
return self._format_single_doc(doc)
if not return_response:
return {"ok": True}
return doc
def _create_comment(self, user, *args, **kwargs):
base_id = '696931697082350_'
generated_id = base_id + str(int(time.time() * 1000))
content = kwargs['content']
channel = kwargs['channel']
parent_id = kwargs.get('parent', None)
# Fill in custom content
R_POST_OBJECTDATA['content'] = content
R_POST_OBJECTDATA['facebook']['_wrapped_data']['message'] = content
R_POST_OBJECTDATA['facebook']['_wrapped_data']['id'] = generated_id
R_POST_OBJECTDATA['facebook']['facebook_post_id'] = generated_id
R_POST_OBJECTDATA['channel'] = channel
if parent_id:
R_POST_OBJECTDATA['facebook']['in_reply_to_status_id'] = parent_id
factory_by_user(user, **R_POST_OBJECTDATA)
return dict(ok=True)
def fb_get_post_comments(channel, user, target, since, until):
from solariat_bottle.db.post.utils import factory_by_user
from solariat_bottle.daemons.facebook.facebook_data_handlers import FBDataHandlerFactory
from solariat_bottle.daemons.facebook.facebook_history_scrapper import FacebookHistoryScrapper
puller = FacebookHistoryScrapper(channel, user)
new_posts = puller.get_posts(since=since, until=until, target=target)
for post in new_posts['data']:
#avoid handling posts with no data useful to parsing
if 'message' not in post and 'link' not in post:
continue
try:
post = puller.handle_data_item(
data=post,
handler=FBDataHandlerFactory.get_instance(
FBDataHandlerFactory.POST),
target_id=target)
except Exception, ex:
logger.error(ex)
continue
comments = post.pop('comments', [])
factory_by_user(user, sync=True, **post)
for comment in comments:
post_id = post['id']
try:
comment = puller.handle_data_item(
data=comment,
handler=FBDataHandlerFactory.get_instance(
FBDataHandlerFactory.COMMENT),
target_id=post_id)
except Exception, ex:
logger.error(ex)
continue
factory_by_user(user, sync=True, **comment)
def fb_get_private_messages(channel, page_id, user, since, until):
from solariat_bottle.db.post.utils import factory_by_user
from solariat_bottle.daemons.facebook.facebook_data_handlers import FBDataHandlerFactory
from solariat_bottle.daemons.facebook.facebook_history_scrapper import FacebookHistoryScrapper
puller = FacebookHistoryScrapper(channel, user=user)
message_threads = puller.get_page_private_messages(page_id, since,
until)['data']
for thread in message_threads:
if 'messages' in thread and thread['messages']['data']:
data = thread['messages'][
'data'][::-1] #get array of messages in reversed order
root_message_id = data[0]['id']
if thread['id'] in channel.tracked_fb_message_threads_ids:
check = lambda msg, since: utc(
parser.parse(msg['created_time'])) > utc(since)
messages_to_handle = [msg for msg in data if check(msg, since)]
else:
channel.tracked_fb_message_threads_ids.append(thread['id'])
channel.save()
messages_to_handle = data
conversation_id = thread['id']
for msg in messages_to_handle:
if msg['id'] != root_message_id:
msg['root_post'] = root_message_id
msg['conversation_id'] = conversation_id
msg['page_id'] = page_id
msg = puller.handle_data_item(
msg,
FBDataHandlerFactory.get_instance(FBDataHandlerFactory.PM),
thread['id'])
factory_by_user(user, sync=True, **msg)
def test_workflow(self):
"""
Testing Predictive Matching coupled with Conversation State Machine
"""
# creating three posts
customer_post = factory_by_user(
self.user,
channels=[str(self.sc.inbound)],
content='Hi, I need a laptop battery',
user_profile=self.chat_profile.customer_profile,
extra_fields={'chat': {
'session_id': 'test_sessin_id'
}})
brand_post = factory_by_user(
self.user,
channels=[str(self.sc.outbound)],
content='Hey, we will ship new battery to you asap',
actor_id=self.agent_profile.id,
is_inbound=False,
extra_fields={
'chat': {
'session_id': 'test_sessin_id',
'in_reply_to_status_id': customer_post.id
}
})
customer_post2 = factory_by_user(
self.user,
channels=[str(self.sc.inbound)],
content="Thank you! That's very nice!",
user_profile=self.customer,
extra_fields={
'chat': {
'session_id': 'test_sessin_id',
'in_reply_to_status_id': brand_post.id
}
})
self.assertEqual(Conversation.objects().count(), 1)
def _create_doc(self, user, *args, **kwargs):
def parse_bool(val):
return val in ('', None, True, 'true', 'True', 'y', 'yes')
add_to_queue = parse_bool(kwargs.pop('add_to_queue', True))
format_docs = parse_bool(kwargs.pop('format_docs', True))
return_response = parse_bool(kwargs.pop('return_response', True))
if {'serialized_to_json', 'post_object_data'} <= set(kwargs.keys()) \
and parse_bool(kwargs['serialized_to_json']):
post_kwargs = json.loads(kwargs['post_object_data'])
else:
post_kwargs = kwargs
post_kwargs.setdefault('add_to_queue', add_to_queue)
post = factory_by_user(user, **post_kwargs)
if not return_response:
return {'ok': True}
if format_docs:
return self._format_single_doc(post)
return post
def create_post(self, **data):
from solariat_bottle.db.post.utils import factory_by_user
return factory_by_user(self.user, **data)
def test_event(self):
# testing PostEvent creation
customer_post = factory_by_user(self.user,
channels=[self.sc.inbound],
content='Hi, I need a laptop battery',
user_profile=self.customer_up)
def create_post(self, **data):
from solariat_bottle.db.post.utils import factory_by_user
from solariat_bottle.db.user import User
user = User.objects.get(email=self.user)
return factory_by_user(user, **data)