def test_integer_id(self):
def make_id_ts_left(time_slot, dummy):
components = (
(time_slot, 22),
(dummy, 42),
)
id_ = pack_components(*components)
return id_
def make_id_ts_right(time_slot, dummy):
components = ((dummy, 42), (time_slot, 22))
id_ = pack_components(*components)
return id_
TimeSlotIntegerId.objects.coll.remove()
from solariat.utils.timeslot import datetime_to_timeslot, parse_date_interval, timedelta, timeslot_to_datetime
start_date, end_date = parse_date_interval('02/21/2013', '05/21/2013')
step = timedelta(hours=24)
dates = []
while start_date < end_date:
dates.append(start_date)
start_date += step
assert len(dates) == 90
data = enumerate(dates[::-1], start=100)
for dummy, date in data:
dummy %= 5
time_slot = datetime_to_timeslot(date)
id_ = make_id_ts_left(time_slot, dummy)
doc = TimeSlotIntegerId(id=id_, time_slot=time_slot, dummy=dummy)
doc.save()
#print list(TimeSlotIntegerId.objects.coll.find())
#fetch interval
start_date, end_date = parse_date_interval('03/21/2013', '04/21/2013')
#start_dummy = 0
#end_dummy = (1L << 41) - 1
start_id = make_id_ts_left(datetime_to_timeslot(start_date, 'hour'), 0)
end_id = make_id_ts_left(datetime_to_timeslot(end_date, 'hour'), 0)
# print start_id.bit_length()
# print end_id.bit_length()
for doc in TimeSlotIntegerId.objects(id__gte=start_id, id__lte=end_id):
print timeslot_to_datetime(doc.time_slot)
self.assertGreaterEqual(doc.time_slot,
datetime_to_timeslot(start_date, 'hour'))
self.assertLessEqual(doc.time_slot,
datetime_to_timeslot(end_date, 'hour'))
def postprocess_params(self, params):
r = params
if 'channel_id' in r:
r['channel'] = get_channels(self.user, r['channel_id'])
set_languages_param(r)
if 'from' in r and 'to' in r:
from_date = r['from']
to_date = r['to'] or from_date
from_dt, to_dt = parse_date_interval(from_date, to_date)
r['from_ts'] = Timeslot(from_dt, r['level'])
r['to_ts'] = Timeslot(to_dt, r['level'])
r['agents'] = get_agents(self.user, r['agents'] or [])
r['statuses'] = get_statuses(r['statuses'], r['plot_type'])
if r['sentiments'] is not None:
assert r[
'intentions'] is None, 'intentions and sentiments cannot be set together'
r['intentions'] = translate_sentiments_to_intentions(
r['sentiments'])
# for some reports we show only problem posts
if is_problem(r['plot_type']):
r['intentions'] = [SATYPE_NAME_TO_ID_MAP['problem']]
# -- cleanup --
del r['channel_id']
r.pop('from', None)
r.pop('to', None)
del r['sentiments']
del r['level']
return params
def test_upsert_feedback(self):
predictor = mock.create_autospec(BasePredictor)
predictor.name = "Social Media Predictor"
entities_registry = EntitiesRegistry()
try:
predictor.upsert_feedback()
except:
self.assertTrue(False)
# mockup upsert_feedback method
expression = 'collect(InfomartEvent)'
expression_data = entities_registry.generate_expression_context(
predictor, expression)
self.assertEqual(expression_data['expression'],
u'collect(predictor, from_dt, to_dt, InfomartEvent)')
_from_dt = datetime.datetime.fromtimestamp(
int(1466964000)).strftime('%m/%d/%Y')
_to_dt = datetime.datetime.fromtimestamp(
int(1467050400)).strftime('%m/%d/%Y')
expression_data['context']['from_dt'], expression_data['context'][
'to_dt'] = parse_date_interval(_from_dt, _to_dt)
parser = BaseParser(expression_data['expression'],
entities_registry.get_all_entries())
parser.evaluate(expression_data['context'])
expression = 'int(123.3333)'
expression_data = entities_registry.generate_expression_context(
predictor, expression)
self.assertEqual(expression_data['expression'], u'int(123.3333)')
parser = BaseParser(expression_data['expression'],
entities_registry.get_all_entries())
result = parser.evaluate(expression_data['context'])
self.assertEqual(result, 123)
expression = 'str(10)'
expression_data = entities_registry.generate_expression_context(
predictor, expression)
parser = BaseParser(expression_data['expression'],
entities_registry.get_all_entries())
result = parser.evaluate(expression_data['context'])
self.assertTrue(isinstance(result, basestring))
def customer_journeys(user):
data = _get_request_data()
customer_id = data['customer_id']
from_ = data['from']
to_ = data['to']
from_dt, to_dt = parse_date_interval(from_, to_)
F = CustomerJourney.F
CustomerProfile = user.account.get_customer_profile_class()
try:
profile = CustomerProfile.objects.get(customer_id)
except CustomerProfile.DoesNotExist:
profile_schema = user.account.customer_profile._get()
profile = profile_schema.get_data_class().objects.get(customer_id)
journeys = CustomerJourney.objects.coll.find(
{
F.customer_id: profile.id,
#F.start_date: {'$gte': from_dt, '$lte': to_dt}
},
{
F.id: 1,
F.journey_type_id: 1,
F.first_event_date: 1,
F.last_event_date: 1
})
journeys_info = []
for j in journeys:
journeys_info.append({
'id':
str(j[F.id]),
'journey_type':
JourneyType.objects.get(j[F.journey_type_id]).display_name,
'start_date':
j[F.first_event_date].isoformat(),
'end_date':
j[F.last_event_date].isoformat(),
'typeId':
str(j[F.journey_type_id])
})
return jsonify(ok=True, journeys=journeys_info)
def get_channel_stats(user):
data = request.json
if not data:
return jsonify(ok=False, error='JSON data is required')
for param in ['channel_id', 'from', 'to', 'level', 'stats_type']:
if not data.get(param):
return jsonify(ok=False,
error='%s is not provided, got %s' % (param, data))
try:
channel = Channel.objects.get_by_user(user, data['channel_id'])
except Channel.DoesNotExist:
return jsonify(ok=False,
error='Channel %(channel_id)s does not exist' % data)
try:
from_dt, to_dt = parse_date_interval(data['from'], data['to'])
except:
return jsonify(
ok=False,
error=
'"from" and "to" parameters should be dates encoded like mm/dd/YY"'
)
level = data['level']
try:
if data['stats_type'] == 'speech_acts':
if 'intention' not in data:
raise RuntimeError('intention is not provided, got %s' % data)
return _get_speech_acts_stats(user, channel, from_dt, to_dt, level,
data['intention'])
else:
return _get_performance_stats(user, channel, from_dt, to_dt, level,
data['stats_type'])
except RuntimeError, exc:
app.logger.error("on getting speech act stats", exc_info=True)
return jsonify(ok=False, error=str(exc))
def service_channel_stats(user):
data = request.json
if not data:
return jsonify(ok=False, error='JSON data is required')
for param in ['from', 'to', 'stats', 'channel_id']:
if not data.get(param):
return jsonify(ok=False,
error='%s is not provided, got %s' %
(param, str(data)))
try:
channel = Channel.objects.get_by_user(user, data['channel_id'])
except Channel.DoesNotExist:
return jsonify(ok=False,
error='Channel %s does not exist' % data['channel_id'])
from_, to_ = parse_date_interval(data['from'], data['to'])
level = guess_timeslot_level(from_, to_)
if not from_ or not to_:
return jsonify(ok=False,
error='"from" and "to" paramater should be provided"')
#valid_stats = set(['volume', 'latency'])
#stats = list(set(data['stats']).intersection(valid_stats))
stats = ['volume', 'latency']
try:
data, counts = aggregate_stats(user, channel, from_, to_, level, stats)
result = dict(
zip(stats, [
dict(data=data[stat], label=stat, count=counts[stat])
for stat in stats
]))
return jsonify(ok=True, data=result)
except RuntimeError, exc:
app.logger.error("on getting speech act stats", exc_info=True)
return jsonify(ok=False, error=str(exc))
#"stats_type":["review","posted","rejected","skipped","filtered","retweeted","clicked","custom","alternate"]
data = request.json
if data is None:
raise abort(415)
if 'channel_id' not in data:
return jsonify(ok=False, error='channel_id should be provided')
try:
channel = Channel.objects.get_by_user(user, data['channel_id'])
except Channel.DoesNotExist, e:
return jsonify(ok=False, error=str(e))
from_dt, to_dt = parse_date_interval(data['from'], data['to'])
level = guess_timeslot_level(from_dt, to_dt)
stats_type = data.get('stats_type', None)
if stats_type:
stats_type = map(str, stats_type)
def _get_statuses(stats):
statuses = []
if set(stats).intersection(['posted', 'custom', 'alternate']):
statuses.append('posted')
if 'review' in stats:
statuses.extend(REVIEW_STATUSES)
statuses.extend(set(stats).intersection(RESPONSE_STATUSES))
def _get_items(user, **kw):
from_, to_ = kw.pop('from', None), kw.pop('to', None)
if from_ and to_:
from_, to_ = parse_date_interval(from_, to_)
else:
from_ = to_ = now()
status = kw.pop('status', None)
if status:
kw['status'] = status
if 'groups' in kw and kw['groups']:
kw['acl'] = kw.pop('groups')
adaptive_learning = kw.pop('adaptive_learning_enabled', None)
if adaptive_learning is not None:
adaptive_learning = parse_bool(adaptive_learning)
#kw['adaptive_learning_enabled'] = adaptive_learning
channel = kw.pop('channel', None)
if channel:
from ..db.channel.base import Channel
try:
channel = Channel.objects.find_by_user(user, id=channel)[0]
except:
return {'ok': False, 'message': 'Channel not found.'}
else:
kw['parent_channel'] = channel.id
items = [
item for item in fetch_items(user, **kw) if adaptive_learning is None
or adaptive_learning == item.adaptive_learning_enabled
]
level = guess_timeslot_level(from_, to_)
item_ids = map(itemgetter('id'), items)
channel_stats_map = aggregate_stats(
user,
item_ids,
from_,
to_,
level,
aggregate=('number_of_posts', 'number_of_false_negative',
'number_of_true_positive', 'number_of_false_positive'))
for k, v in channel_stats_map.items():
if v:
denominator = v['number_of_true_positive'] + v[
'number_of_false_positive']
if denominator:
v['presicion'] = float(
v['number_of_true_positive']) / denominator
denominator = v['number_of_true_positive'] + v[
'number_of_false_negative']
if denominator:
v['recall'] = float(v['number_of_true_positive']) / denominator
channel_stats_map[k] = v
def _to_dict(x):
obj = _item_to_dict(x, user)
obj.update({"stats": channel_stats_map.get(str(x.id))})
return obj
try:
return dict(ok=True,
size=len(items),
list=[_to_dict(x) for x in items])
except Exception, e:
app.logger.error(e)
def list_channels(user):
data = request.json or {}
account = parse_account(user, data)
try:
results = []
if data.get('widget'):
channels = filtered_channels(find_channels(user, account, status='Active'))
for channel in channels:
results.append({
"id": str(channel.id),
"title": channel.title})
elif data.get('primitive'):
channels = filtered_channels(find_channels(user, account),
filter_service=True,
filter_compound=True)
for channel in channels:
results.append({
"id": str(channel.id),
"title": channel.title,
"platform": channel.platform,
"is_dispatchable": channel.is_dispatchable})
elif data.get('service'):
channels = find_channels(user, account, channel_class=ServiceChannel)
for channel in channels:
results.append({
"id": str(channel.id),
"title": channel.title})
else:
channels = filtered_channels(find_channels(user, account))
if data.get('outbound') or request.args.get('outbound'):
channels = [ch for ch in channels if ch.is_dispatchable]
channel_stats_map = {}
if data.get('stats'):
from_, to_ = parse_date_interval(data.get('from'), data.get('to'))
level = guess_timeslot_level(from_, to_)
from operator import itemgetter
channel_ids = map(itemgetter('id'), channels)
channel_stats_map = aggregate_stats(
user, channel_ids, from_, to_, level,
aggregate=(
'number_of_posts',
'number_of_false_positive',
'number_of_true_positive',
'number_of_false_negative'
)
)
for channel in channels:
stats = channel_stats_map.get(str(channel.id))
channel_dict = {
"id" : str(channel.id),
"title" : channel.title,
"type_name" : channel.__class__.__name__,
"type" : "channel",
"parent" : str(channel.parent_channel or ""),
"status" : channel.status,
"description" : channel.description,
"created_at" : datetime_to_timestamp_ms(channel.created),
"platform" : channel.platform,
"account" : channel.account and channel.account.name,
"perm" : 'r' if channel.read_only else channel.perms(user),
"facebook_page_ids" : channel.facebook_page_ids if hasattr(channel, 'facebook_page_ids') else None,
"facebook_access_token" : channel.facebook_access_token if hasattr(channel, 'facebook_access_token') else None,
"access_token_key" : channel.access_token_key if hasattr(channel, 'access_token_key') else None,
"access_token_secret" : channel.access_token_secret if hasattr(channel, 'access_token_secret') else None,
"twitter_handle" : channel.twitter_handle if hasattr(channel, 'twitter_handle') else None,
"is_compound" : channel.is_compound,
"stats" : stats}
results.append(channel_dict)
results.sort(key=lambda x: x['title'])
return jsonify(ok=True, list=results)
except Exception, exc:
app.logger.error('error on /channels/json',
exc_info=True)
return jsonify(ok=False, error=str(exc))
def accounts_handler(user):
"""
Accounts CRUD for superuser.
"""
if not (user.is_staff or user.is_admin):
if request.method != 'GET':
params = request.json
account = user.account
existing_selected_app = account.selected_app
new_selected_app = params.get('selected_app')
if new_selected_app and new_selected_app != existing_selected_app:
# Update just to be safe we're up to date with any changes in CONFIGURABLE_APPS
available_apps = account.available_apps
for key in available_apps.keys():
if key in CONFIGURABLE_APPS:
available_apps[key] = CONFIGURABLE_APPS[key]
else:
available_apps.pop(key)
account.available_apps = available_apps
account.selected_app = new_selected_app
account.save()
return jsonify(ok=True)
else:
return jsonify(ok=False,
error='No access to editing accounts.')
else:
#return all accounts accessible by non-privileged user
return jsonify(ok=True,
data=[
_json_account(acct,
user,
with_stats=request.args.get(
'stats', False))
for acct in user.available_accounts
])
if request.method == 'GET':
from_, to_ = None, None
if request.args.get('account_id'):
return account_details_handler()
with_stats = request.args.get('stats', False)
if with_stats:
try:
from_, to_ = parse_date_interval(
request.args.get('from', None),
request.args.get('to', None))
except:
pass
return jsonify(ok=True,
data=_query_accounts(user,
with_stats=with_stats,
start_date=from_,
end_date=to_))
elif request.method == 'POST':
params = request.json
# Need to check whether the account exists
name = params.get('name')
if not name:
return jsonify(ok=False, error="Name is required.")
exists = Account.objects.find(name=name).count()
if exists:
ok = False
result = "Account name '{}' already exists.".format(name)
#ok, result = _update_account(user, **params)
else:
ok, result = _create_account(user, params)
if ok:
return jsonify(ok=ok, account=_json_account(result, user))
else:
return jsonify(ok=ok, error=result)
elif request.method == 'DELETE':
data = request.args
if not data:
return jsonify(ok=False, error='no parameters provided')
acct_id = data.get('id')
if acct_id:
try:
account = Account.objects.get_by_user(user, acct_id)
res, msg = _delete_account(user, account)
if not res:
raise AppException(msg)
except AppException, ex:
return jsonify(ok=False, error=str(ex))
else:
return jsonify(ok=True, data={})
else:
return jsonify(ok=False, error="No account id")