class FacebookRateLimitInfo(Document):
access_token = fields.StringField()
failed_request_time = fields.DateTimeField()
error_code = fields.NumField(null=True, choices=FB_RATE_LIMIT_ERRORS + [None])
path = fields.StringField()
wait_until = fields.DateTimeField()
channel = fields.StringField()
log_item = fields.ObjectIdField()
indexes = [('token', 'error_code')]
manager = FacebookRateLimitInfoManager
LIMITS_CONFIG = {
THROTTLING_USER: BackOffStrategy(30*60, 30*60, 1.0),
THROTTLING_APP: BackOffStrategy(225, 60*60, 2.0),
ERROR_MISUSE: BackOffStrategy(60 * 60, 24 * 60 * 60, 3.0),
THROTTLING_API_PATH: BackOffStrategy(60, 60*60, 2.0)
}
@property
def wait_time(self):
return (utc(self.wait_until) - utc(self.failed_request_time)).total_seconds()
@property
def remaining_time(self):
return (utc(self.wait_until) - now()).total_seconds()
@property
def exc(self):
return FacebookRateLimitError(
code=self.error_code,
remaining_time=self.remaining_time,
path=self.path)
class _ScheduledTaskDoc(Document):
collection = 'ScheduledTask'
started_time = fields.DateTimeField()
last_run = fields.DateTimeField()
next_run = fields.DateTimeField()
_interval = fields.NumField()
state = fields.NumField()
def get_interval(self):
return timedelta(milliseconds=int(self._interval) * 1000)
def __init__(self, data=None, **kwargs):
interval = kwargs.get('interval', None)
kwargs.pop('interval', None)
super(_ScheduledTaskDoc, self).__init__(data, **kwargs)
if data is None:
if not isinstance(interval, timedelta):
raise Exception(
"Interval should be an instance of 'timedelta'")
self._interval = interval.total_seconds()
self.started_time = init_with_default(kwargs, 'started_time',
datetime.utcnow())
self.last_run = init_with_default(kwargs, 'last_run',
self.started_time)
self.state = init_with_default(kwargs, 'state',
TaskStateEnum.WAIT_NEXT)
self.set_next_run()
def set_next_run(self):
self.last_run = datetime.utcnow()
self.next_run = self.last_run + self.get_interval()
self.save()
class FooBar(Document):
name = fields.StringField(db_field='nm')
status = fields.StringField(db_field='stts', choices=['active', 'deactivated', 'suspended'])
counter = fields.NumField(db_field='cntr')
created_at = fields.DateTimeField(db_field='crtd')
updated_at = fields.DateTimeField(db_field='updtd')
active = fields.BooleanField(db_field='actv')
stages = fields.ListField(fields.StringField(), db_field='stgs')
class FacebookUserMixin(object):
_cached_facebook_me = fields.StringField(db_field='fb_me')
_cached_facebook_me_ts = fields.DateTimeField(db_field='fb_me_ts')
# Cache channel description on GSA side and update once per day
_cached_channel_description = fields.StringField()
_cached_last_description_update = fields.DateTimeField(default=datetime.now())
def _set_channel_description(self, channel_info):
self._cached_channel_description = json.dumps(channel_info)
self._cached_last_description_update = datetime.now()
self.save()
def _get_channel_description(self):
if self._cached_channel_description:
if self._cached_last_description_update + timedelta(days=1) < datetime.now():
# After 1 day just consider this to be too old and basically invalid
self._cached_channel_description = ""
self.save()
return None
# We're still in 'safe' 1 day range, return cached value
try:
return json.loads(self._cached_channel_description)
except Exception:
return None
return None
channel_description = property(_get_channel_description, _set_channel_description)
def facebook_me(self, force=False):
default_timeout = 60 * 60 # 1 hour
def _graph_me():
graph = get_facebook_api(self)
return graph.request('/me')
def _invalidate(timeout=default_timeout,
value_attr='_cached_facebook_me',
ts_attr='_cached_facebook_me_ts',
value_getter=_graph_me):
date_now = now()
if not getattr(self, ts_attr) or (date_now - utc(getattr(self, ts_attr))).total_seconds() > timeout:
self.update(**{ts_attr: date_now,
value_attr: json.dumps(value_getter())})
return json.loads(getattr(self, value_attr))
timeout = 0 if force is True else default_timeout
return _invalidate(timeout)
def set_facebook_me(self, fb_user):
self.update(_cached_facebook_me_ts=now(),
_cached_facebook_me=json.dumps(fb_user))
class StreamLog(Document):
"""Created on streamref creation, updated on stream stops"""
accounts = fields.ListField(fields.ObjectIdField())
channels = fields.ListField(fields.ObjectIdField())
stream_ref_id = fields.BytesField()
started_at = fields.DateTimeField(null=True)
stopped_at = fields.DateTimeField(null=True)
indexes = [('accounts', ), ('channels', ), ('stream_ref_id', )]
class FacebookRequestLog(Document):
channel = fields.ObjectIdField(null=True, db_field='cl')
access_token = fields.StringField(db_field='tok')
path = fields.StringField(db_field='uri')
method = fields.StringField(db_field='m')
args = fields.StringField(db_field='arg')
post_args = fields.StringField(db_field='parg')
start_time = fields.DateTimeField(db_field='ts')
end_time = fields.DateTimeField(db_field='et')
elapsed = fields.NumField(db_field='el')
error = fields.StringField(db_field='er', null=True)
indexes = [('start_time', 'access_token', 'path')]
manager = FacebookRequestLogManager
class QueueMessage(Document):
manager = QueueMessageManager
channel_id = fields.ListField(fields.StringField())
created_at = fields.DateTimeField()
reserved_until = fields.DateTimeField()
post_data = fields.DictField()
batch_token = fields.StringField()
indexes = [
('channel_id', 'reserved_until'),
('batch_token', ),
]
class JourneyStage(ABCPredictor):
collection = 'JourneyStage'
manager = JourneyStageManager
journey_id = fields.ObjectIdField(db_field='jo')
stage_type_id = fields.ObjectIdField(
db_field='st') # Will be reference to a JourneyStageType
stage_name = fields.StringField(db_field='sn')
effort_info = fields.DictField(
db_field='ef'
) # Embedded doc with any effort info we're going to track
reward_info = fields.DictField(
db_field='ri'
) # Embedded doc with any reward info we're going to track
start_date = fields.DateTimeField(db_field='sd')
end_date = fields.DateTimeField(db_field='ed')
last_updated = fields.DateTimeField(
db_field='lu'
) # We're probably going to want to know when was the last even from this stage directly
last_event = fields.EventIdField(
db_field='le') # Keep track of the event itself
def check_preconditions(self, event):
if hasattr(event, 'stage_id') and event.stage_id:
# If 'stage_id' exists on event, and it's not None, that will be a precondition and acceptance rule
return str(self.id) == event.stage_id
if hasattr(event, 'journeys') and event.journeys:
# If a specific set of journeys was passed with the event, the journey for this stage
return self.journey_id in event.journeys
return True
def rule_based_match(self, object):
if hasattr(object, 'stage_id') and object.stage_id:
# If 'stage_id' exists on event, and it's not None, that will be a precondition and acceptance rule
return str(self.id) == object.stage_id
return False
def process_event(self, event):
update_dict = dict(set__last_event=event.data['_id'],
set__last_updated=event.datetime_from_id)
self.update(**update_dict)
def get_journey_stage_type(self):
from solariat_bottle.db.journeys.journey_type import JourneyStageType
return JourneyStageType.objects.get(self.stage_type_id)
class ContactLabel(AuthDocument):
admin_roles = [ADMIN, STAFF, ANALYST]
manager = ContactLabelManager
title = fields.StringField(db_field='te')
created = fields.DateTimeField(db_field='cd', default=datetime.utcnow())
platform = fields.StringField(db_field='pm')
status = fields.StringField(db_field='st')
users = fields.ListField(fields.StringField())
allow_inheritance = True
@classmethod
def class_based_access(cls, account):
""" Based on the AUTH class we are creating, we might offer some default access
to certain groups from the account. By default, permissions should only be given to
admin type users. This can be overwritten in specific classes as needed. E.G. messages -> agents ?
"""
if account is None:
return []
return [
default_admin_group(account),
default_analyst_group(account),
default_reviewer_group(account)
]
@property
def type_id(self):
return 0
class FacetCache(AuthDocument):
collection = "FacetCache"
hashcode = fields.StringField(db_field='hc', required=True)
page_type = fields.StringField(db_field='pe', required=True)
account_id = fields.ObjectIdField(db_field='aid', required=True)
value = fields.StringField(db_field='ve', required=True)
created_at = fields.DateTimeField(db_field='ct', required=True)
def is_up_to_date(self):
delta = datetime.now() - self.created_at
return delta.total_seconds() < get_var(
'MONGO_CACHE_EXPIRATION') # 30 mins
@classmethod
def upsert_cache_record(cls, hashcode, data, page_type, account_id):
now = datetime.now()
if 'time_stats' in data:
del data['time_stats']
if 'pipelines' in data:
del data['pipelines']
cache_records = FacetCache.objects(hashcode=hashcode,
account_id=account_id,
page_type=page_type)
cache_records_num = cache_records.count()
if cache_records_num >= 2:
raise Exception('Too many cache records')
elif cache_records_num == 1:
cache = cache_records[0]
cache.value = json.dumps(data)
cache.created_at = datetime.now()
cache.save()
else:
cache = FacetCache(hashcode=hashcode,
value=json.dumps(data),
account_id=account_id,
page_type=page_type,
created_at=now)
cache.save()
@classmethod
def get_cache(cls, params, account_id, page_type):
hash_arg = params.copy()
# import ipdb; ipdb.set_trace()
for field in ['force_recompute', 'range_alias']:
if field in hash_arg:
del hash_arg[field]
hashcode = md5(str(hash_arg)).hexdigest()
cache_candidates = FacetCache.objects(hashcode=hashcode,
account_id=account_id,
page_type=page_type)
if not cache_candidates:
return hashcode, None
elif 1 == len(cache_candidates):
return hashcode, cache_candidates[0]
else:
for cache in cache_candidates:
cache.remove()
return hashcode, None
class AgentProfile(DynamicProfile):
pass
cardinalities_lu = fields.DateTimeField()
has_indexes = fields.BooleanField(default=False)
refresh_rate = timedelta(hours=2)
def compute_cardinalities(self):
if self.cardinalities_lu:
if self.cardinalities_lu + self.refresh_rate >= datetime.utcnow():
return
super(AgentProfile, self).compute_cardinalities()
if self.cardinalities:
self.cardinalities_lu = datetime.utcnow()
if not self.has_indexes:
self.create_indexes()
self.save()
def create_indexes(self):
coll = self.data_coll
# For bigger collections try and create indexes
if self.id_field:
coll.create_index(self.id_field)
for key, value in self.cardinalities.iteritems():
try:
coll.create_index(key)
except OperationFailure, ex:
LOGGER.warning(
"Mongo operation failed while trying to create index: %s",
ex)
if self.created_at_field:
coll.create_index(self.created_at_field)
self.has_indexes = True
class NPSPost(Post):
PROFILE_CLASS = NPSProfile
manager = NPSPostManager
allow_inheritance = True
collection = "NPSPost"
case_number = fields.StringField(db_field='cn', required=True)
last_modified = fields.DateTimeField(db_field='lm', required=False)
case_update_id = fields.StringField(db_field='cd', required=False)
type = fields.StringField(db_field='te', required=False)
native_id = fields.StringField(db_field='nd', required=True)
description = fields.StringField(db_field='dsc')
survey_response_name = fields.StringField(db_field='srn')
@classmethod
def gen_id(cls,
account,
actor_id,
_created,
in_reply_to_native_id,
parent_event=None):
CustomerProfile = account.get_customer_profile_class()
actor_num = CustomerProfile.objects.get(id=actor_id).actor_num
packed = pack_event_id(actor_num, _created)
return packed
@property
def computed_tags(self):
return list(
set(self._computed_tags +
[str(smt.id)
for smt in self.accepted_smart_tags] + self.assigned_tags))
class TaskMessage(Document):
'''
Internal Structure representing the integartion
data structure with a data stream provider.
'''
_created = fields.DateTimeField(db_field='ca', default=datetime.now())
content = fields.StringField(db_field='ct', required=True)
type = fields.StringField(db_field='tp', required=True)
user = fields.ListField(fields.ReferenceField(User))
manager = TaskMessageManager
def add_item(self):
''' Increment counters'''
self._update_item(1)
def remove_item(self):
''' Decrement counters or remove if empty '''
if self.entry_count >= 2:
self._update_item(-1)
else:
self.delete()
def set_datasift_hash(self, datasift_hash):
" set atomically datasift hash and update last_sync "
return self.objects.coll.find_and_modify(
query={'_id': self.id},
update={
'$set': {
self.fields['datasift_hash'].db_field: datasift_hash,
self.fields['last_sync'].db_field: datetime.now()
}
},
new=True)
class PostFilter(Document):
'''
Internal Structure representing the integartion
data structure with a data stream provider.
'''
filter_type_id = fields.NumField(db_field='fd', choices=FILTER_TYPE_IDS)
# How many entries
entry_count = fields.NumField(db_field='et', default=0)
# How many more entries can you handle
spare_capacity = fields.NumField(db_field='sy',
default=POSTFILTER_CAPACITY)
datasift_hash = fields.StringField(db_field='dh')
last_update = fields.DateTimeField(db_field='lu', default=datetime.now())
last_sync = fields.DateTimeField(db_field='ls')
def _update_item(self, n):
self.update(inc__entry_count=n,
inc__spare_capacity=-n,
set__last_update=datetime.now())
def add_item(self):
''' Increment counters'''
self._update_item(1)
def remove_item(self):
''' Decrement counters or remove if empty '''
if self.entry_count >= 2:
self._update_item(-1)
else:
self.delete()
def set_datasift_hash(self, datasift_hash):
" set atomically datasift hash and update last_sync "
return self.objects.coll.find_and_modify(
query={'_id': self.id},
update={
'$set': {
self.fields['datasift_hash'].db_field: datasift_hash,
self.fields['last_sync'].db_field: datetime.now()
}
},
new=True)
class BaseProfile(AuthDocument):
manager = ProfileManager
allow_inheritance = True
collection = "BaseProfiles"
account_id = fields.ObjectIdField()
first_name = fields.StringField()
last_name = fields.StringField()
age = fields.NumField()
sex = fields.StringField()
location = fields.StringField()
seniority = fields.StringField()
assigned_labels = fields.ListField(fields.ObjectIdField())
date_of_birth = fields.StringField()
attached_data = fields.DictField()
products = fields.ListField(fields.StringField())
actor_num = AutoIncrementField(counter_name='ActorCounter', db_field='ar')
created_at = fields.DateTimeField(default=now)
linked_profile_ids = fields.ListField(fields.StringField())
indexes = ['actor_num', 'linked_profile_ids']
@property
def linked_profiles(self):
from solariat_bottle.db.user_profiles.user_profile import UserProfile
return UserProfile.objects(id__in=self.linked_profile_ids)[:]
def get_profile_of_type(self, typename):
if not isinstance(typename, basestring):
typename = typename.__name__
for profile in self.linked_profiles:
if profile.__class__.__name__ == typename:
return profile
def add_profile(self, profile):
new_id = str(profile.id)
if new_id not in self.linked_profile_ids:
self.linked_profile_ids.append(new_id)
self.update(addToSet__linked_profile_ids=new_id)
def get_age(self):
# Best guess we can make is by date of birth if present and properly formatted
if self.date_of_birth:
try:
dob = datetime.strptime(self.date_of_birth, AGE_FORMAT)
return relativedelta(datetime.now(), dob).years
except Exception, ex:
LOGGER.error(ex)
# Next, if actual age is present, use that but also store updated dob
if self.age:
dob = datetime.now() - relativedelta(years=self.age)
self.date_of_birth = dob.strftime(AGE_FORMAT)
self.save()
return self.age
return None
class EventTag(ABCPredictor):
indexes = [('account_id', 'is_multi', ), ]
display_name = fields.StringField()
account_id = fields.ObjectIdField()
status = fields.StringField(default="Active")
description = fields.StringField()
created = fields.DateTimeField()
channels = fields.ListField(fields.ObjectIdField())
manager = EventTagManager
default_threshold = 0.49
@property
def inclusion_threshold(self):
return self.default_threshold
def save(self):
self.packed_clf = self.clf.packed_model
super(EventTag, self).save()
def match(self, event):
assert isinstance(event, Event), "EventTag expects Event objects"
if self.score(event) > self.inclusion_threshold:
return True
return False
def score(self, event):
assert isinstance(event, Event), "EventTag expects Event objects"
return super(EventTag, self).score(event)
def accept(self, event):
assert isinstance(event, Event), "EventTag expects Event objects"
return super(EventTag, self).accept(event)
def reject(self, event):
assert isinstance(event, Event), "EventTag expects Event objects"
return super(EventTag, self).reject(event)
def check_preconditions(self, event):
if self.precondition:
return eval(self.precondition)
return self.feature_extractor.check_preconditions(event, self.features_metadata)
def rule_based_match(self, event):
if self.acceptance_rule:
return eval(self.acceptance_rule)
return False
def to_dict(self, fields_to_show=None):
result_dict = super(EventTag, self).to_dict()
result_dict.pop('counter')
result_dict.pop('packed_clf')
result_dict['channels'] = [str(c) for c in result_dict['channels']]
return result_dict
class WidgetModel(Document):
"""
A WidgetModel is a abstract widget that can be instantiated to ConcreteWidget
and used in corresponding typed dashboard
"""
title = fields.StringField(required=True, unique=True)
description = fields.StringField()
settings = fields.DictField()
created = fields.DateTimeField(default=datetime.now)
class Gallery(Document):
"""
A gallery is dashboard_type specific, will contain collection of predifined
widget models
"""
dashboard_type = fields.ReferenceField(DashboardType, required=True)
widget_models = fields.ListField(fields.ReferenceField(WidgetModel))
created = fields.DateTimeField(default=datetime.now)
def to_dict(self):
rv = super(Gallery, self).to_dict()
rv['display_name'] = self.dashboard_type.display_name
rv['type'] = self.dashboard_type.type
return rv
class Funnel(AuthDocument):
"""
"""
name = fields.StringField(required=True, unique=True)
description = fields.StringField()
journey_type = fields.ObjectIdField()
steps = fields.ListField(fields.ObjectIdField(), required=True)
owner = fields.ReferenceField(User)
created = fields.DateTimeField(default=datetime.now)
def to_dict(self, fields_to_show=None):
rv = super(Funnel, self).to_dict()
rv['steps'] = map(str, self.steps)
return rv
class PredictorModel(Document):
collection = 'PredictorModel'
allow_inheritance = True
version = fields.NumField()
predictor = fields.ReferenceField('BasePredictor')
parent = fields.ObjectIdField()
weight = fields.NumField()
display_name = fields.StringField()
description = fields.StringField()
# is_active = fields.BooleanField(default=False)
task_data = fields.EmbeddedDocumentField(TaskData)
last_run = fields.DateTimeField()
context_features = fields.ListField(fields.DictField())
action_features = fields.ListField(fields.DictField())
train_data_percentage = fields.NumField(default=80)
n_rows = fields.NumField()
min_samples_thresould = fields.NumField(default=1)
from_dt = fields.DateTimeField()
to_dt = fields.DateTimeField()
def score(self, *args, **kwargs):
pass
def feedback(self, *args, **kwargs):
pass
def search(self, *args, **kwargs):
pass
def to_json(self, *args, **kwargs):
from solariat_bottle.db.predictors.base_predictor import PredictorConfigurationConversion
data = super(PredictorModel, self).to_json(*args, **kwargs)
data = PredictorConfigurationConversion.python_to_json(data)
return data
class BaseScore(AuthDocument):
created = fields.DateTimeField(default=now)
matching_engine = fields.ObjectIdField()
model_id = fields.ObjectIdField(null=True)
counter = fields.NumField(default=1)
cumulative_latency = fields.NumField(required=True)
indexes = [
('matching_engine', 'created'),
]
@property
def latency(self):
return 1.0 * self.cumulative_latency / (self.counter or 1)
class DashboardWidget(Document):
'''
Internal Structure representing the integartion
data structure with a data stream provider.
'''
created = fields.DateTimeField(db_field='c', default=datetime.now)
settings = fields.DictField(db_field='s')
order = fields.NumField(db_field='o')
title = fields.StringField(db_field='t', required=True)
user = fields.ReferenceField(User, db_field='u')
dashboard_id = fields.ObjectIdField(required=True)
manager = DashboardWidgetManager
def to_dict(self):
base_dict = dict(title=self.title,
order=self.order,
id=str(self.id),
dashboard_id=str(self.dashboard_id))
base_dict.update(self.settings)
return base_dict
def copy_to(self, dashboard):
new_widget_data = {
'title': self.title,
'user': dashboard.owner,
'dashboard_id': dashboard.id,
}
new_widget_data.update(self.settings)
widget = DashboardWidget.objects.create_by_user(**new_widget_data)
return widget
def delete(self):
dashboard = Dashboard.objects.get_by_user(self.user,
id=self.dashboard_id)
dashboard._remove_widget(self)
super(DashboardWidget, self).delete()
def __repr__(self):
return "<DashboardWidget: %s; id: %s>" % (self.title, self.id)
class DashboardType(Document):
collection = 'DashboardType'
manager = DashboardTypeManager
type = fields.StringField(required=True, unique=True)
display_name = fields.StringField(required=True, unique=True)
owner = fields.ReferenceField(User)
created = fields.DateTimeField(default=datetime.now)
def __repr__(self):
return "<DashboardType: %s; id: %s>" % (self.display_name, self.id)
def to_dict(self):
rv = super(DashboardType, self).to_dict()
if self.owner:
rv['owner_name'] = '%s %s' % (self.owner.first_name
or '', self.owner.last_name or '')
rv['email'] = self.owner.email
else:
rv['owner_name'] = ''
rv['email'] = ''
return rv
class BaseFeedback(AuthDocument):
created = fields.DateTimeField(default=now)
action = fields.DictField()
context = fields.DictField()
matching_engine = fields.ObjectIdField()
model_id = fields.ObjectIdField(null=True)
reward = fields.NumField()
# predicted score
est_reward = fields.NumField()
context_vector = fields.DictField()
action_vector = fields.DictField()
# scoring latency in ms
score_runtime = fields.NumField() # time taken in millisecond to compute score
# scoring error %
score_diff = fields.NumField() # (reward - score) / reward
indexes = [('matching_engine', 'created'), ]
class TaskData(SonDocument):
updated_at = fields.DateTimeField()
total = fields.NumField()
done = fields.NumField()
@property
def status(self):
return 'training' if 0 < self.done < self.total else 'idle'
@property
def progress(self):
return int(100 * (self.done or 0.0) / (self.total or 1.0))
def to_dict(self, fields_to_show=None):
task = self
json_data = super(TaskData, self).to_dict(fields_to_show)
json_data.update(
progress=self.progress,
status=task.status,
updated_at=task.updated_at and str(task.updated_at)
)
return json_data
class Dashboard(AuthDocument):
collection = 'Dashboard'
manager = DashboardManager
type_id = fields.ObjectIdField(required=True)
title = fields.StringField(required=True)
description = fields.StringField()
owner = fields.ReferenceField(User)
author = fields.ReferenceField(User)
widgets = fields.ListField(fields.ObjectIdField())
shared_to = fields.ListField(fields.ObjectIdField())
filters = fields.DictField()
created = fields.DateTimeField(default=datetime.now)
admin_roles = {STAFF, ADMIN, REVIEWER, ANALYST}
def to_dict(self, fields_to_show=None):
rv = super(Dashboard, self).to_dict()
rv['widgets'] = map(str, self.widgets)
rv['shared_to'] = map(str, self.shared_to)
rv['owner_name'] = '%s %s' % (self.owner.first_name
or '', self.owner.last_name or '')
rv['author_name'] = '%s %s' % (self.author.first_name
or '', self.author.last_name or '')
rv['owner_email'] = self.owner.email
rv['author_email'] = self.author.email
rv['account_id'] = str(self.owner.account.id)
rv['type'] = DashboardType.objects.get(self.type_id).type
return rv
def __repr__(self):
return "<Dashboard: %s; id: %s>" % (self.title, self.id)
def _add_widget(self, widget):
"""
"""
self.widgets.append(widget.id)
self.save()
def _remove_widget(self, widget):
"""
widget is not automatically deleted. To delete, use `.delete_widget()` instead.
`widget.dashboard_id` will still point to this dashboard.
"""
self.widgets.remove(widget.id)
self.save()
def delete_widget(self, widget):
if isinstance(widget, (basestring, fields.ObjectId)):
widget = DashboardWidget.objects.get(widget)
widget.delete()
def delete(self):
for widget_id in self.widgets:
self.delete_widget(widget_id)
super(Dashboard, self).delete_by_user(self.owner)
def copy_to(self, user, title=None, description=None):
dashboard_data = {
'type_id': self.type_id,
'title': title or self.title,
'description': description or self.description,
'author': self.owner,
'owner': user,
'widgets': [],
'shared_to': [],
'filters': self.filters,
}
# FIX: create_by_user is having role error
dashboard = Dashboard.objects.create_by_user(user, **dashboard_data)
#dashboard = Dashboard.objects.create(**dashboard_data)
for widget_id in self.widgets:
widget = DashboardWidget.objects.get(widget_id)
widget.copy_to(dashboard)
return dashboard
class BrokenQueueMessage(QueueMessage):
manager = BrokenQueueMessageManager
persisted_at = fields.DateTimeField(default=datetime.utcnow)
class TimeMark(SonDocument):
PERIODIC_DATE_FORMAT = '%m-%d'
STATIC_DATE_FORMAT = '%Y-%m-%d'
TIME_FORMAT = '%H:%M:%S'
DAY_OF_WEEK = 0
PERIODIC_DATE = 1
STATIC_DATE = 2
TYPE_NAME_MAP = {
DAY_OF_WEEK: 'dayofweek',
PERIODIC_DATE: 'periodicdate',
STATIC_DATE: 'staticdate'}
TYPE_CLASS_MAP = property(lambda self: {
self.DAY_OF_WEEK: DayOfWeek,
self.PERIODIC_DATE: PeriodicDate,
self.STATIC_DATE: StaticDate})
mark_type = fields.NumField(choices=(DAY_OF_WEEK, PERIODIC_DATE, STATIC_DATE), db_field='t')
value = fields.StringField()
_from_time = fields.DateTimeField(db_field='a', default=None, null=True)
_to_time = fields.DateTimeField(db_field='b', default=None, null=True)
@property
def from_time(self):
return self._from_time and self._from_time.time() or time()
@property
def from_time_fmt(self):
return self._from_time and self.from_time.strftime(self.TIME_FORMAT)
@property
def to_time(self):
return self._to_time and self._to_time.time() or time(23, 59, 59, 999999)
@property
def to_time_fmt(self):
return self._to_time and self.to_time.strftime(self.TIME_FORMAT)
@property
def mark_type_str(self):
return self.TYPE_NAME_MAP[self.mark_type]
def __init__(self, data=None, **kw):
if self.__class__ != TimeMark and not ('mark_type' in kw or data and 't' in data):
m = inverse(self.TYPE_CLASS_MAP)
kw.update(mark_type=m[self.__class__])
super(TimeMark, self).__init__(data=data, **kw)
class_ = self.TYPE_CLASS_MAP[self.data['t']]
if class_ != self.__class__:
self.__class__ = class_
class_.validate(self)
def __hash__(self):
return (self.mark_type, self.value,
self.from_time_fmt, self.to_time_fmt)
def __str__(self):
return u"<%s %s %s %s>" % (self.__class__.__name__, self.value,
self.from_time_fmt, self.to_time_fmt)
def __eq__(self, other):
return self.__hash__() == other.__hash__()
def __lt__(self, other):
raise TypeError("%s is not comparable type" % self.__class__.__name__)
def to_json(self, fields_to_show=None):
result = {
"type": self.mark_type_str,
"from": self.from_time_fmt,
"to": self.to_time_fmt,
"value": self.value
}
return result
@classmethod
def from_json(cls, data):
m = inverse(cls.TYPE_NAME_MAP)
type_ = m[data['type']]
from_time = None
to_time = None
if data['from']:
from_time = datetime.strptime(data['from'], cls.TIME_FORMAT)
if data['to']:
to_time = datetime.strptime(data['to'], cls.TIME_FORMAT)
return cls(
mark_type=type_,
value=data['value'],
_from_time=from_time,
_to_time=to_time
)
def time_points(self):
return self.from_time, self.to_time
def validate(self):
assert self.from_time < self.to_time, "from_time must be less than to_time"
class JobStatus(ArchivingAuthDocument):
STATUSES = PENDING, RUNNING, ABANDONED, SUCCESSFUL, FAILED, \
RESUBMITTED, SLEEPING, TERMINATED = \
'Pending', 'Running', 'Abandoned', 'Successful', 'Failed', \
'Resubmitted', 'Sleeping', 'Terminated'
RUNNABLE_STATUSES = PENDING, SLEEPING
collection = 'jobs'
account = fields.ObjectIdField(null=True)
topic = fields.StringField()
name = fields.StringField()
args = fields.PickledField()
kwargs = fields.PickledField()
metadata = fields.DictField(null=True)
created_at = fields.DateTimeField()
started_date = fields.DateTimeField()
completion_date = fields.DateTimeField()
status = fields.StringField(choices=STATUSES)
state = fields.DictField()
last_activity = fields.DateTimeField()
awake_at = fields.DateTimeField(null=True)
@property
def git_commit(self):
return (self.metadata or {}).get('git_commit')
@property
def resubmission_info(self):
return (self.metadata or {}).get('resubmitted')
def abandon(self):
if self.status == self.PENDING:
self.status = self.ABANDONED
res = self.objects.coll.update(
{
self.F.id: self.id,
self.F.status: self.PENDING
}, {"$set": {
self.F.status: self.ABANDONED
}})
if isinstance(res, dict) and res.get('nModified') == 1:
return True
def resume(self):
if self.status != self.FAILED:
raise RuntimeError("Job can not be resumed in '{}' state.".format(
self.status))
from solariat_bottle.jobs.manager import manager
job = manager.registry.get(self.name)
res = job.submit(self.topic, self.name, self.args, self.kwargs,
self.metadata)
# updating old job
meta = self.metadata or {}
meta.update(resubmitted={
'new_id': res.job_instance.id,
'result': str(res.submission_result)
})
self.update(status=JobStatus.RESUBMITTED, metadata=meta)
# updating new job
meta = res.job_instance.metadata or {}
meta.update(resubmitted={'old_id': self.id})
res.job_instance.update(metadata=meta)
return [self, res.job_instance]
def can_edit(self, user_or_group, admin_roles=None):
if admin_roles is None:
admin_roles = self.admin_roles
account_check = user_or_group.is_staff or (user_or_group.is_admin
and user_or_group.account.id
== self.account)
edit_check = (bool(
set(admin_roles).intersection(set(user_or_group.user_roles)))
or (hasattr(user_or_group, 'is_superuser')
and user_or_group.is_superuser))
return account_check and edit_check
@property
def wait_time(self):
if self.started_date and self.created_at:
return (utc(self.started_date or now()) -
utc(self.created_at)).total_seconds()
@property
def execution_time(self):
if self.completion_date and self.started_date:
now_ = now()
return (utc(self.completion_date or now_) -
utc(self.started_date or now_)).total_seconds()
class CustomerJourney(AuthDocument, EventSequenceStatsMixin):
FEAT_TYPE = 'type'
FEAT_LABEL = 'label'
FEAT_EXPR = 'field_expr'
FEAT_NAME = 'name'
collection = "CustomerJourney"
manager = CustomerJourneyManager
stage_name = fields.StringField(
db_field='fs') # stage_name of current_stage
# Dict in the form:
# <strategy_type> : <list of index__stage_name>. strategy_type can be for now (default, platform, event_type)
stage_sequences = fields.DictField(db_field='sseq')
# Dict in the form
# index__stage_name: {actual attributes computed for this specific stage}
stage_information = fields.DictField(db_field='si')
customer_id = fields.BaseField(
db_field='ci') # dynamic profiles may use custom id type
customer_name = fields.StringField(
db_field='cn') # Just for quick access w/o extra db call
agent_ids = fields.ListField(fields.ObjectIdField(), db_field='ag')
agent_names = fields.ListField(
fields.StringField(),
db_field='ans') # Just for quick access w/o extra db calls
journey_tags = fields.ListField(fields.ObjectIdField(), db_field='jts')
channels = fields.ListField(fields.ObjectIdField(), db_field='chls')
last_updated = fields.DateTimeField(db_field='lu')
# time spent by events in each stage-eventtype status
node_sequence = fields.ListField(fields.DictField(), db_field='nds')
node_sequence_agr = fields.ListField(fields.StringField(), db_field='ndsn')
# time spent by events in each stage-eventtype status
journey_attributes_schema = fields.ListField(fields.DictField(),
db_field='jas')
first_event_date = fields.DateTimeField(db_field='fed')
last_event_date = fields.DateTimeField(db_field='led')
indexes = [('journey_type_id', 'journey_tags'), ('journey_attributes', ),
('journey_type_id', 'channels'), ('customer_id', ),
('agent_ids', )]
parsers_cache = dict()
@classmethod
def to_mongo(cls, data, fill_defaults=True):
"""
Same as super method, except parser.evaluate is skipped (would be called in process_event)
"""
return super(CustomerJourney,
cls).to_mongo(data,
fill_defaults=fill_defaults,
evaluate=False)
@classmethod
def metric_label(cls, metric, param, value):
# from solariat_bottle.db.predictors.customer_segment import CustomerSegment
if param == 'status':
value = JourneyStageType.STATUS_TEXT_MAP[value]
if param == 'journey_type_id':
value = JourneyType.objects.get(value).display_name
#value = value[0] if type(value) in [list, tuple] and value else value if value is not None else 'N/A'
if value is None:
value = 'N/A'
return str(value)
def ui_repr(self):
base_repr = "Status: %s; Start date: %s; End date: %s;" % (
self.status, self.first_event_date, self.last_event_date)
if self.customer_name:
base_repr += " Customer: %s;" % self.customer_name
if self.agent_names:
base_repr += " Agents: %s;" % self.agent_names
return base_repr
def to_dict(self, *args, **kwargs):
# from solariat_bottle.db.predictors.customer_segment import CustomerSegment
base_dict = super(CustomerJourney, self).to_dict()
base_dict['agents'] = map(str, self.agents)
base_dict['channels'] = map(str, self.channels)
base_dict['smart_tags'] = map(str, self.smart_tags)
base_dict['journey_tags'] = map(str, self.journey_tags)
base_dict['status'] = JourneyStageType.STATUS_TEXT_MAP[self.status]
base_dict['string_repr'] = self.ui_repr()
base_dict['journey_attributes'] = self.journey_attributes
return base_dict
def handle_add_tag(self, tag_id):
tag_id = ObjectId(tag_id)
self.update(addToSet__smart_tags=tag_id)
def handle_remove_tag(self, tag_id):
tag_id = ObjectId(tag_id)
self.update(pull__smart_tags=tag_id)
def apply_schema(self, expression, context):
hash_key = str(expression) + '__'.join(context)
if hash_key in CustomerJourney.parsers_cache:
parser = CustomerJourney.parsers_cache[hash_key]
else:
parser = BaseParser(expression, context.keys())
CustomerJourney.parsers_cache[hash_key] = parser
try:
value = parser.evaluate(context)
except TypeError:
value = None
return value
def process_event(self, event, customer, agent, journey_stage_type):
self._current_event = event
received_event_from_past = False
created_at = utc(event.created_at)
last_updated = utc(self.last_updated) if self.last_updated else None
if last_updated and created_at < last_updated:
# log.error("=========RECEIVED EVENT FROM THE PAST %s %s < last updated %s" % (
# event, event.created_at, self.last_updated))
received_event_from_past = True
# IMPORTANT: No mongo calls should be done here at all!
if agent:
if agent.id not in self.agent_ids:
self.agent_ids.append(agent.id)
# TODO: This needs to be enforced on profile dynamic classes as a separate specific
# column (can be optional)
self.agent_names.append(str(agent))
# TODO: Same as for agent profile, this needs to be set on dynamic class level
self.customer_name = str(customer)
if event.channels[0] not in self.channels:
self.channels.append(event.channels[0])
if not received_event_from_past:
if journey_stage_type:
self.status = journey_stage_type.status
self.last_event_date = event.created_at
self.last_updated = event.created_at
# TODO: This whole strategy switch will need to be changed to be defined somehow on journey level
# TODO: ISSSUE for the last stage the information is not copied. Will need to do this on journey closure.
for strategy in [
STRATEGY_DEFAULT, STRATEGY_PLATFORM, STRATEGY_EVENT_TYPE
]:
self.check_for_stage_transition(strategy, event,
journey_stage_type)
schema_computed_attributes = dict()
# All of these need to be returned directly from customer data (no extra mongo calls!)
expression_context = dict(agents=self.agents,
customer_profile=self.customer_profile,
current_event=event,
event_sequence=self.event_sequence,
current_stage=self.current_stage,
previous_stage=self.previous_stage,
stage_sequence=self.stage_sequence)
# for k in self.field_names:
# expression_context[k] = getattr(self, k)
# adding func with @property decorator to context
for key in CustomerJourney.get_properties():
expression_context[key] = getattr(self, key)
for schema_entry in self.journey_attributes_schema:
expression = schema_entry[self.FEAT_EXPR]
f_name = schema_entry[self.FEAT_NAME]
schema_computed_attributes[f_name] = self.apply_schema(
expression, expression_context)
expression_context[f_name] = schema_computed_attributes[f_name]
self.journey_attributes = schema_computed_attributes
if self.status in [
JourneyStageType.COMPLETED, JourneyStageType.TERMINATED
]:
self.node_sequence_agr = []
for i, item in enumerate(self.node_sequence):
key, value = item.items()[0]
self.node_sequence_agr.append(key)
@classmethod
def get_properties(cls):
""" returns all list of member funcs decorated with @property """
from copy import deepcopy
base = deepcopy(cls.field_names)
base = [
field for field in base
if field not in ('is_archived', '_t', 'acl', 'match_expression',
'journey_type_id', 'display_name', 'account_id',
'id', 'available_stages')
]
base.extend([
name for name, value in vars(cls).items()
if isinstance(value, property)
])
return base
@property
def CONSTANT_DATE_NOW(self):
# For being picked up by context
from datetime import datetime
return datetime.now()
@property
def CONSTANT_ONE_DAYS(self):
from datetime import timedelta
return timedelta(hours=24)
@property
def CONSTANT_ONE_HOUR(self):
from datetime import timedelta
return timedelta(hours=1)
def check_for_stage_transition(self, strategy, event, journey_stage_type):
current_stage = self.get_current_stage(strategy)
if strategy == STRATEGY_DEFAULT:
new_stage = journey_stage_type.display_name if journey_stage_type else current_stage
elif strategy == STRATEGY_EVENT_TYPE:
new_stage = journey_stage_type.display_name + ':' + str(
event.event_type) if journey_stage_type else current_stage
elif strategy == STRATEGY_PLATFORM:
new_stage = journey_stage_type.display_name + ':' + event.platform if journey_stage_type else current_stage
if new_stage != current_stage:
stage_index = self.get_current_index(strategy)
new_stage_value = STAGE_INDEX_SEPARATOR.join(
[new_stage, str(stage_index + 1)])
if current_stage is not None:
full_stage_name = STAGE_INDEX_SEPARATOR.join(
[current_stage, str(stage_index)])
self.stage_information[
full_stage_name] = self.compute_stage_information(strategy)
self.stage_sequences[strategy] = self.stage_sequences.get(
strategy, []) + [new_stage_value]
if strategy == STRATEGY_DEFAULT:
self.stage_name = journey_stage_type.display_name
self.stage_sequence_names.append(new_stage)
if strategy == STRATEGY_EVENT_TYPE:
if current_stage is None and new_stage is None:
return
# TODO: This is still kind of hard coded for MPC
if new_stage != current_stage:
self.node_sequence.append({new_stage: 1})
else:
self.node_sequence[-1][new_stage] += 1
def compute_stage_information(self, strategy):
info = dict()
for key, val in self.journey_attributes.iteritems():
info[key] = val
info['end_date'] = self.last_event_date
if len(self.stage_sequences.get(strategy, [])) <= 1:
info['start_date'] = self.first_event_date
else:
info['start_date'] = self.stage_information[
self.stage_sequences[strategy][-2]]['end_date']
return info
def close_journey(self):
for strategy in [
STRATEGY_DEFAULT, STRATEGY_PLATFORM, STRATEGY_EVENT_TYPE
]:
current_stage = self.get_current_stage(strategy)
stage_index = self.get_current_index(strategy)
if current_stage is not None:
full_stage_name = STAGE_INDEX_SEPARATOR.join(
[current_stage, str(stage_index)])
self.stage_information[
full_stage_name] = self.compute_stage_information(strategy)
self.save()
def get_current_stage(self, strategy_type):
if not self.stage_sequences.get(strategy_type):
return None
else:
return self.stage_sequences.get(strategy_type)[-1].split(
STAGE_INDEX_SEPARATOR)[0]
def get_current_index(self, strategy_type):
if not self.stage_sequences.get(strategy_type):
return -1
else:
return int(
self.stage_sequences.get(strategy_type)[-1].split(
STAGE_INDEX_SEPARATOR)[1])
def stage_sequence_by_strategy(self, strategy):
return [
val.split(STAGE_INDEX_SEPARATOR)[0]
for val in self.stage_sequences[strategy]
]
def __get_agents(self):
if hasattr(self, '_agents'):
return self._agents
else:
self._agents = self.account.get_agent_profile_class().objects.find(
id__in=self.agent_ids)[:]
return self._agents
def __set_agents(self, agents):
self._agents = agents
agents = property(__get_agents, __set_agents)
def __get_customer_profile(self):
if hasattr(self, '_customer_profile'):
return self._customer_profile
else:
self._customer_profile = self.account.get_customer_profile_class(
).objects.get(self.customer_id)
return self._customer_profile
def __set_customer_profile(self, customer_profile):
self._customer_profile = customer_profile
customer_profile = property(__get_customer_profile, __set_customer_profile)
def __get_current_event(self):
if hasattr(self, '_current_event'):
return self._current_event
else:
self._current_event = self.event_sequence[
-1] if self.event_sequence else None
return self._current_event
def __set_current_event(self, event):
self._current_event = event
current_event = property(__get_current_event, __set_current_event)
def __get_event_sequence(self):
if hasattr(self, '_event_sequence'):
return self._event_sequence
else:
from solariat_bottle.db.account import Account
account = Account.objects.get(self.account_id)
CustomerProfile = account.get_customer_profile_class()
try:
customer = CustomerProfile.objects.get(self.customer_id)
except CustomerProfile.DoesNotExist:
self._event_sequence = []
return self._event_sequence
if self.first_event_date and self.last_event_date:
events = Event.objects.events_for_actor(
self.first_event_date, self.last_event_date,
customer.actor_num)[:]
self._event_sequence = events
return self._event_sequence
# event_type_ids = [x.event_type for x in events]
# event_types = EventType.objects(id__in=event_type_ids)[:]
# event_type_map = {str(x.id): x.name for x in event_types}
# return [event_type_map[x.event_type] for x in events]
self._event_sequence = []
return self._event_sequence
def __set_event_sequence(self, event_sequence):
self._event_sequence = event_sequence
event_sequence = property(__get_event_sequence, __set_event_sequence)
@property
def current_stage(self):
if len(self.stage_sequences.get(STRATEGY_DEFAULT, [])) == 0:
return None
else:
last_stage = self.stage_sequences[STRATEGY_DEFAULT][-1].split(
STAGE_INDEX_SEPARATOR)[0]
return last_stage
@property
def nps(self):
nps1 = self.journey_attributes.get('nps')
event = self.current_event
from solariat_bottle.db.post.nps import NPSOutcome
if isinstance(event, NPSOutcome):
nps2 = self.current_event.score
else:
nps2 = None
return max(nps1, nps2)
@staticmethod
def nps_value_to_label(value):
if value is None:
return 'n/a'
elif 0 <= value <= 6:
return 'detractor'
elif value in (7, 8):
return 'passive'
elif value in (9, 10):
return 'promoter'
else:
raise Exception("invalid nps value (%r given)" % value)
@property
def nps_category(self):
# from solariat_bottle.views.facets import nps_value_to_label
if self.nps == 'N/A':
return 'N/A'
else:
return self.nps_value_to_label(self.nps)
@property
def previous_stage(self):
if self.current_stage is None:
return None
if len(self.stage_sequences.get(STRATEGY_DEFAULT, [])) <= 1:
return None
else:
last_stage = self.stage_sequences[STRATEGY_DEFAULT][-2].split(
STAGE_INDEX_SEPARATOR)[0]
return last_stage
@property
def stage_sequence(self):
if len(self.stage_sequences.get(STRATEGY_DEFAULT, [])) == 0:
return []
else:
return [
val.split(STAGE_INDEX_SEPARATOR)[0]
for val in self.stage_sequences[STRATEGY_DEFAULT]
]
@property
def first_event(self):
event_sequence = self.event_sequence
if event_sequence:
return event_sequence[0]
else:
return None
@property
def is_abandoned(self):
if self.status == JourneyStageType.TERMINATED:
return 1
else:
return 0
@property
def days(self):
if self.first_event_date and self.last_event_date:
return (utc(self.last_event_date) -
utc(self.first_event_date)).days
else:
return None
