def list(self, *args, **kwargs):
account_id = self.user.account.id
platform = kwargs.get(KEY_PLATFORM)
# list event types for choosen channel when importing events
if platform:
if platform in StaticEventType.EVENT_TYPES:
return [STATIC_CHANNEL_EVENT_TYPE_DATA]
event_types = BaseEventType.objects.find_by_user(
self.user, account_id=account_id, platform=platform)
return [
et.to_dict(fields2show=('id', 'name', 'sync_status'))
for et in event_types
]
# list event_types for journey type stages
if kwargs.get('show_all'):
event_types = BaseEventType.objects.find_by_user(
self.user, account_id=account_id)
return [et.to_dict() for et in event_types]
# list dynamic event types
from_dt = to_dt = None
if 'from' in kwargs:
from_dt = timeslot.parse_datetime(kwargs['from'])
to_dt = timeslot.parse_datetime(kwargs['to'])
date_query = get_query_by_dates(from_dt, to_dt)
event_types = EventType.objects.find_by_user(
self.user, parent_id=self.user.account.id, **date_query)
return [s.to_dict() for s in event_types]
def postprocess_params(self, params):
r = params
from_dt = parse_datetime(r['from'])
to_dt = parse_datetime(r['to'])
r['from_ts'] = datetime_to_timeslot(from_dt, 'hour')
r['to_ts'] = datetime_to_timeslot(to_dt, 'hour')
del r['from']
del r['to']
r = super(PostsView, self).postprocess_params(r)
if r['sort_by'] == 'time':
r['sort_map'] = {'_created': -1}
else:
r['sort_map'] = {'intention_confidence': -1}
r['min_conf'] = r['thresholds']['intention']
if r['last_query_time'] is not None:
r['last_query_time'] = timestamp_ms_to_datetime(
r['last_query_time'])
del r['plot_type']
return params
def list(self, *args, **kwargs):
user = self.user
acc = user.account
from_dt = to_dt = None
if 'from' in kwargs:
from_dt = timeslot.parse_datetime(kwargs['from'])
to_dt = timeslot.parse_datetime(kwargs['to'])
datasets = acc.datasets.get_all_datasets(user, from_dt, to_dt)
return [s.to_dict() for s in datasets]
def validate_date_range(data):
start_date = parse_datetime(data.get('start_date'), default=now())
end_date = parse_datetime(data.get('end_date'), default=UNIX_EPOCH)
if not date_is_empty(end_date) and end_date <= start_date:
return {"ok": False, "error": ERROR_DATE_RANGE}
else:
data['start_date'] = start_date
data['end_date'] = end_date
return {"ok": True}
def list(self, *args, **kwargs):
from_dt = to_dt = None
if 'from' in kwargs:
from_dt = timeslot.parse_datetime(kwargs['from'])
to_dt = timeslot.parse_datetime(kwargs['to'])
date_query = get_query_by_dates(from_dt, to_dt)
channel_types = ChannelType.objects.find_by_user(
self.user, account=self.user.account, **date_query)
return [s.to_dict() for s in channel_types]
def postprocess_params(data):
filters = {}
if data['accounts']:
filters['account__in'] = to_list(
data['accounts'], lambda x: Account.objects.get(x).id)
if data['names']:
filters['name__in'] = to_list(data['names'])
if data['status']:
filters['status__in'] = to_list(data['status'])
if data['from']:
filters['created_at__gte'] = parse_datetime(data['from'])
if data['to']:
filters['created_at__lte'] = parse_datetime(data['to'])
return filters, get_paging_params(data)
def test_fetch_by_date(self):
# check start_date. let fetcher made 2 requests (max resp len: 200)
one_sec = timedelta(seconds=1)
FakeTwitterApi.restore_settings()
api = FakeTwitterApi()
_, start_date = api.DM[300]
start_date += one_sec
LOGGER.debug('++++++ DATES: (%s)', (start_date, api.DM[200][1], api.DM[100][1]))
res = DirectMessagesFetcher(api, **{"start_date": start_date})
statuses = list(res.fetch())
self.assertEqual(len(statuses), 300)
# checking end_date filter
FakeTwitterApi.restore_settings()
api = FakeTwitterApi()
_, start_date = api.DM[300]
_, end_date = api.DM[150]
start_date += one_sec
res = DirectMessagesFetcher(api, **{'start_date': start_date, 'end_date': end_date})
statuses = list(res.fetch())
self.assertEqual(len(statuses), 150)
from solariat.utils.timeslot import parse_datetime
self.assertTrue(all(start_date <= parse_datetime(s['created_at']) <= end_date for s in statuses))
def _search(self, user, *args, **kwargs):
error_desc = "Requests should contain a channel id and some query text: {'channel': <>, 'query': <>}"
required_params = ['channel', 'query']
LOGGER.debug('Searching for FAQs with params: %s' % kwargs)
for entry in required_params:
if entry not in kwargs:
error_msg = "Missing required parameter '%s'" % entry
raise exc.InvalidParameterConfiguration(error_msg,
description=error_desc)
try:
channel = Channel.objects.get(kwargs['channel'])
except Channel.DoesNotExist:
raise exc.ResourceDoesNotExist(
"No channel with id=%s found in the system." %
kwargs['channel'])
result = FAQ.objects.text_search(channel, kwargs['query'], limit=100)
LOGGER.debug('Search results for FAQs: %s' % result)
from solariat.utils.timeslot import parse_datetime, now
_created = parse_datetime(kwargs.get('_created', now()), default=now())
faq_event = FAQQueryEvent.objects.create_by_user(user,
query=kwargs['query'],
channels=[channel.id],
_created=_created)
if 'customer_id' in kwargs:
CustomerProfile = user.account.get_customer_profile_class()
customer = CustomerProfile.objects.get(kwargs['customer_id'])
# import ipdb
# ipdb.set_trace()
# actions = NextBestActionEngine.upsert(account_id=customer.account_id).search(faq_event, customer)
# TODO: THink how we can implement this properly with new advisors API
return dict(list=result,
event=faq_event.to_dict()) #, actions=actions)
else:
return dict(list=result, event=faq_event.to_dict())
def validate_dates(data):
date_ = data.get('end_date', UNIX_EPOCH.strftime(DATE_FMT))
if isinstance(date_, dt.datetime):
date_ = date_.strftime(DATE_FMT)
#LOGGER.debug("Validating {} type".format(type(date_)))
data['end_date'] = parse_datetime(date_, default=UNIX_EPOCH)
return data
def _filter_tweet(self, tweet):
predicates = []
p = predicates.append
tweet_date = parse_datetime(tweet['created_at'])
start_date = self.filters.get('start_date')
end_date = self.filters.get('end_date')
if start_date:
p(tweet_date >= utc(start_date))
if end_date:
p(tweet_date <= utc(end_date))
return all(predicates)
def test_sort_by_and_date_range(self):
self.create_responses()
[resp1, resp2, resp3, resp4] = list(Response.objects.find())
filter_ = {'intentions': ['asks', 'needs', 'likes', 'problem']}
resp4.post_date = parse_datetime('2010-01-01 00:00:00')
resp4.relevance = 9
resp4.intention_confidence = 0.9
resp4.save()
resp3.post_date = parse_datetime('2010-01-03 00:00:00')
resp3.intention_confidence = 0.99
resp3.relevance = 8
resp3.save()
resp2.post_date = parse_datetime('2010-01-04 00:00:00')
resp2.relevance = 7
resp2.intention_confidence = 0.8
resp2.save()
resp1.relevance = 6
resp1.post_date = parse_datetime('2010-01-05 00:00:00')
resp1.intention_confidence = 0.7
resp1.save()
for r in Response.objects.find():
print r.post.plaintext_content, r.intention_confidence
self.assertFalse(resp1.to_ui(self.user)['has_history'])
self.assertFalse(resp1.to_ui(self.user)['has_more_matches'])
filter_['sort_by'] = 'time'
resp_dicts = self._get_response_dicts(filter_)
self.assertEqual(resp_dicts[0]['id'], str(resp1.id))
# filter_['sort_by'] = 'relevance'
# resp_dicts = self._get_response_dicts(filter_)
# self.assertEqual(resp_dicts[0]['id'], str(resp4.id))
return
'''
def patch_post_kw(cls, kw, native_data=None):
if not kw.get('_created'):
created_at = native_data and cls.platform_created_at(native_data) or None
created_time = parse_datetime(created_at, default=now())
# some facebook entities like images may have created_at=1999-01-01T08:00:00 +0000
# which is before TIMESLOT_EPOCH and causes AssertError in pack_event_id
if created_time <= TIMESLOT_EPOCH:
wrapped_data = native_data.get('_wrapped_data', {})
if not isinstance(wrapped_data, dict):
try:
wrapped_data = json.loads(wrapped_data)
except (ValueError, TypeError):
wrapped_data = {}
updated_at = native_data.get('updated_time') or wrapped_data.get('updated_time', None)
created_time = parse_datetime(updated_at, default=now())
if created_time < TIMESLOT_EPOCH:
created_time = now()
kw['_created'] = created_time
channel = kw.get('channel') or kw['channels'][0]
channel = channel if isinstance(channel, Channel) else Channel.objects.get(id=channel)
# account = channel.account
user_profile = kw.get('user_profile')
# TODO: [gsejop] is_inbound should not be a posted property,
# event is routed to a set of channels
# Apparently, is_inbound and actor_id are just extra helping information
# sent from jop data generation script
kw['is_inbound'] = kw.pop('is_inbound', channel.is_inbound)
# if not kw.get('actor_id', False):
# if kw['is_inbound'] == True:
# kw['actor_id'] = user_profile.customer_profile.id
# elif kw['is_inbound'] == False:
# kw['actor_id'] = user_profile.agent_profile.id
# else:
# raise Exception('is_inbound should be set')
kw.setdefault('actor_id', user_profile.id)
return kw
def push_posts(self, tweets, post_data_format=None, post_data_dump=None):
insert_data = dict(subscription=self.subscription,
post_data_format=post_data_format)
for tweet in tweets:
log_state(self.subscription.channel.id, str(tweet['id']),
PostState.ARRIVED_IN_RECOVERY)
try:
insert_data.update({
"post_data":
post_data_dump(tweet),
"timestamp":
datetime_to_timestamp(parse_datetime(tweet['created_at']))
})
QueuedHistoricData.objects.create(**insert_data)
log_state(self.subscription.channel.id, str(tweet['id']),
PostState.ADDED_TO_WORKER_QUEUE)
except:
LOGGER.exception(u'QueuedHistoricData.objects.create: %s' %
insert_data)
return len(tweets)
def create_by_user(self, user, **kw):
safe_create = kw.pop('safe_create', False)
if not safe_create:
raise AppException("Use db.post.utils.factory_by_user instead")
add_to_queue = kw.pop('add_to_queue', False)
sync = kw.pop('sync',
False) # We might consider dropping this entirely
self.doc_class.patch_post_kw(kw)
# handling extra_fields
chat_data = kw.pop('chat_data', None)
kw.setdefault("extra_fields", {})
if chat_data:
kw["extra_fields"].update({"chat": chat_data})
kw["extra_fields"].setdefault("chat", {})
session_id = kw.get(
"session_id", None) or kw["extra_fields"]["chat"].get("session_id")
if not session_id:
session_id = self.gen_session_id()
kw["session_id"] = session_id
chat_created_at = chat_data.get('created_at',
None) if chat_data else dict()
if chat_created_at:
kw['_created'] = utc(parse_datetime(chat_created_at))
assert 'actor_id' in kw, "No 'actor_id' provided with chat message, could not infer it based on " + str(
kw)
assert 'is_inbound' in kw, "No 'is_inbound' provided with chat message, could not infer it based on " + str(
kw)
CustomerProfile = user.account.get_customer_profile_class()
AgentProfile = user.account.get_agent_profile_class()
if 'user_profile' not in kw: # If we have customer id but no specific profile, try to find it in our system
if kw['is_inbound']:
customer_or_agent = CustomerProfile.objects.get(kw['actor_id'])
else:
customer_or_agent = AgentProfile.objects.get(kw['actor_id'])
profile = customer_or_agent.get_profile_of_type(ChatProfile)
if not profile:
profile = ChatProfile.anonymous_profile(platform='Chat')
kw['user_profile'] = profile
if not kw['is_inbound']:
# We know it's outbound post, we need to figure out actor id based on parent from chat session
try:
parent = self.doc_class.objects.find(
session_id=session_id,
is_inbound=True).sort(_created=-1).limit(1)[:][0]
# if we can't figure it out, let's put untracked post as a parent
except IndexError:
parent = UntrackedChatPost()
kw['_id'] = pack_event_id(parent.actor.actor_num, kw['_created'])
else:
actor_num = self.doc_class.get_actor(True,
kw['actor_id']).actor_num
kw['_id'] = pack_event_id(actor_num, kw['_created'])
# We know that it's inbound post, but may be the first post in conversation was outbound.
# If that's the case, then this outbound post was fired by UntrackedProfile
# Now we can encode id using current CustomerProfile instead of UntrackedProfile
outbount_events = self.doc_class.objects.find(
session_id=session_id, is_inbound=False)[:]
for e in outbount_events:
parent_actor_num, dt = unpack_event_id(e.id)
if parent_actor_num == 0:
e.delete()
e.id = pack_event_id(actor_num, dt)
e.save()
kw['force_create'] = True
lang_data = kw.pop('lang', Language(('en', 1)))
# creation
post = self.create(**kw)
# postprocess_new_post(user, post) - failing for now, something with tag assignments
assert post.session_id, "ChatPost should have chat session_id"
self._set_post_lang(post, lang_data)
postprocess_new_post(user, post, add_to_queue)
get_service_channel(post.channel).post_received(post)
return post
def get_timerange_level(self):
try:
return guess_timeslot_level(parse_datetime(self.filters['from']), parse_datetime(self.filters['to']))
except:
LOGGER.warn('Unknown period to determine the timerange level')
def parse_created_at(self):
_created_at = self.platform_created_at(self.native_data)
return parse_datetime(_created_at, default=utc(now()))
def _email_created_at(self):
return parse_datetime(self._email_data.get('created_at', None))
def _parse_date(created_at):
if isinstance(created_at, (date, datetime)):
return utc(created_at)
return parse_datetime(created_at)
def parse_created_at(self):
return utc(parse_datetime(
self._chat_created_at)) if self._chat_created_at else None
def post(self, user, *args, **kwargs):
"""
:param channel: REQUIRED, the id of the channel we want to create a historic load for
:param from_date: REQUIRED, the start date of the load, format is '%d.%m.%Y-%H:%M:%S'
:param to_date: REQUIRED, the end date of the load, format is '%d.%m.%Y-%H:%M:%S'
:returns A json response, format: {ok=<True|False>, error="", item=subscription.json}
Sample response (Twitter):
{
"ok": true,
"subscription": {
"status": "created",
"channel_id": "541aec9731eddd1fc4507853",
"from_date": 1404388800,
"to_date": 1405252800,
"datasift_historic_id": null,
"datasift_push_id": null,
"id": "541aec9731eddd1fc4507857"
}
}
Sample response (Facebook):
{
"ok": true,
"subscription": {
"status": "created",
"finished": [],
"actionable": [],
"channel_id": "541aeccd31eddd1fec5b2059",
"from_date": 1404388800,
"to_date": 1405252800,
"id": "541aeccd31eddd1fec5b205c"
}
}
"""
if 'channel' not in kwargs:
exc_err = "Parameter 'channel' is required in order to create subscription."
raise exc.InvalidParameterConfiguration(exc_err)
channel = self.get_channel(user, kwargs)
if 'from_date' not in kwargs:
raise exc.InvalidParameterConfiguration(
"Parameter 'from_date' required")
if 'to_date' not in kwargs:
raise exc.InvalidParameterConfiguration(
"Parameter 'to_date' required")
from_date = parse_datetime(kwargs['from_date'])
to_date = parse_datetime(kwargs['to_date'])
if from_date >= to_date:
raise exc.InvalidParameterConfiguration(
"'From' date must be less than 'To' date")
type = kwargs.get('type', '')
if type == 'event' and not channel.tracked_fb_event_ids:
raise exc.InvalidParameterConfiguration(
"No Events set, canceling recovery.")
sc = get_service_channel(channel)
if not isinstance(sc, FacebookServiceChannel):
subscription_cls = HistoricalSubscriptionFactory.resolve(sc)
if subscription_cls is None:
raise exc.InvalidParameterConfiguration(
"Could not infer a service channel for channel %s" %
(channel.title + '<' + channel.__class__.__name__ + '>'))
error = subscription_cls.validate_recovery_range(
from_date, to_date)
if error:
raise exc.InvalidParameterConfiguration(error)
if not kwargs.pop('force',
False) and subscription_cls.objects.find_one(
channel_id=sc.id, status__in=STATUS_ACTIVE):
raise exc.ForbiddenOperation(
"The recovery process is already in progress for this channel"
)
subscription = subscription_cls.objects.create(
created_by=user,
channel_id=sc.id,
from_date=from_date,
to_date=to_date,
status=SUBSCRIPTION_CREATED)
if not get_var('APP_MODE') == 'test':
subscription.process()
return dict(ok=True, subscription=subscription.to_dict())
else:
if isinstance(kwargs['from_date'], int) and isinstance(
kwargs['to_date'], int):
from_date_str = int(kwargs['from_date']) / 1000
to_date_str = int(kwargs['to_date']) / 1000
else:
from_date_str = from_date.strftime("%s")
to_date_str = to_date.strftime("%s")
url = "%s?token=%s&channel=%s&since=%s&until=%s&type=%s" % \
(settings.FBOT_URL + '/json/restore', settings.FB_DEFAULT_TOKEN, sc.id, from_date_str, to_date_str, type)
from solariat_bottle.tasks import async_requests
async_requests.ignore('get', url, verify=False, timeout=None)
return dict(ok=True)
def dates_cmp(date1, date2):
from solariat.utils.timeslot import parse_datetime
return cmp(parse_datetime(date1), parse_datetime(date2))