class FacebookRateLimitInfo(Document): access_token = fields.StringField() failed_request_time = fields.DateTimeField() error_code = fields.NumField(null=True, choices=FB_RATE_LIMIT_ERRORS + [None]) path = fields.StringField() wait_until = fields.DateTimeField() channel = fields.StringField() log_item = fields.ObjectIdField() indexes = [('token', 'error_code')] manager = FacebookRateLimitInfoManager LIMITS_CONFIG = { THROTTLING_USER: BackOffStrategy(30*60, 30*60, 1.0), THROTTLING_APP: BackOffStrategy(225, 60*60, 2.0), ERROR_MISUSE: BackOffStrategy(60 * 60, 24 * 60 * 60, 3.0), THROTTLING_API_PATH: BackOffStrategy(60, 60*60, 2.0) } @property def wait_time(self): return (utc(self.wait_until) - utc(self.failed_request_time)).total_seconds() @property def remaining_time(self): return (utc(self.wait_until) - now()).total_seconds() @property def exc(self): return FacebookRateLimitError( code=self.error_code, remaining_time=self.remaining_time, path=self.path)
class _ScheduledTaskDoc(Document): collection = 'ScheduledTask' started_time = fields.DateTimeField() last_run = fields.DateTimeField() next_run = fields.DateTimeField() _interval = fields.NumField() state = fields.NumField() def get_interval(self): return timedelta(milliseconds=int(self._interval) * 1000) def __init__(self, data=None, **kwargs): interval = kwargs.get('interval', None) kwargs.pop('interval', None) super(_ScheduledTaskDoc, self).__init__(data, **kwargs) if data is None: if not isinstance(interval, timedelta): raise Exception( "Interval should be an instance of 'timedelta'") self._interval = interval.total_seconds() self.started_time = init_with_default(kwargs, 'started_time', datetime.utcnow()) self.last_run = init_with_default(kwargs, 'last_run', self.started_time) self.state = init_with_default(kwargs, 'state', TaskStateEnum.WAIT_NEXT) self.set_next_run() def set_next_run(self): self.last_run = datetime.utcnow() self.next_run = self.last_run + self.get_interval() self.save()
class FooBar(Document): name = fields.StringField(db_field='nm') status = fields.StringField(db_field='stts', choices=['active', 'deactivated', 'suspended']) counter = fields.NumField(db_field='cntr') created_at = fields.DateTimeField(db_field='crtd') updated_at = fields.DateTimeField(db_field='updtd') active = fields.BooleanField(db_field='actv') stages = fields.ListField(fields.StringField(), db_field='stgs')
class FacebookUserMixin(object): _cached_facebook_me = fields.StringField(db_field='fb_me') _cached_facebook_me_ts = fields.DateTimeField(db_field='fb_me_ts') # Cache channel description on GSA side and update once per day _cached_channel_description = fields.StringField() _cached_last_description_update = fields.DateTimeField(default=datetime.now()) def _set_channel_description(self, channel_info): self._cached_channel_description = json.dumps(channel_info) self._cached_last_description_update = datetime.now() self.save() def _get_channel_description(self): if self._cached_channel_description: if self._cached_last_description_update + timedelta(days=1) < datetime.now(): # After 1 day just consider this to be too old and basically invalid self._cached_channel_description = "" self.save() return None # We're still in 'safe' 1 day range, return cached value try: return json.loads(self._cached_channel_description) except Exception: return None return None channel_description = property(_get_channel_description, _set_channel_description) def facebook_me(self, force=False): default_timeout = 60 * 60 # 1 hour def _graph_me(): graph = get_facebook_api(self) return graph.request('/me') def _invalidate(timeout=default_timeout, value_attr='_cached_facebook_me', ts_attr='_cached_facebook_me_ts', value_getter=_graph_me): date_now = now() if not getattr(self, ts_attr) or (date_now - utc(getattr(self, ts_attr))).total_seconds() > timeout: self.update(**{ts_attr: date_now, value_attr: json.dumps(value_getter())}) return json.loads(getattr(self, value_attr)) timeout = 0 if force is True else default_timeout return _invalidate(timeout) def set_facebook_me(self, fb_user): self.update(_cached_facebook_me_ts=now(), _cached_facebook_me=json.dumps(fb_user))
class StreamLog(Document): """Created on streamref creation, updated on stream stops""" accounts = fields.ListField(fields.ObjectIdField()) channels = fields.ListField(fields.ObjectIdField()) stream_ref_id = fields.BytesField() started_at = fields.DateTimeField(null=True) stopped_at = fields.DateTimeField(null=True) indexes = [('accounts', ), ('channels', ), ('stream_ref_id', )]
class FacebookRequestLog(Document): channel = fields.ObjectIdField(null=True, db_field='cl') access_token = fields.StringField(db_field='tok') path = fields.StringField(db_field='uri') method = fields.StringField(db_field='m') args = fields.StringField(db_field='arg') post_args = fields.StringField(db_field='parg') start_time = fields.DateTimeField(db_field='ts') end_time = fields.DateTimeField(db_field='et') elapsed = fields.NumField(db_field='el') error = fields.StringField(db_field='er', null=True) indexes = [('start_time', 'access_token', 'path')] manager = FacebookRequestLogManager
class QueueMessage(Document): manager = QueueMessageManager channel_id = fields.ListField(fields.StringField()) created_at = fields.DateTimeField() reserved_until = fields.DateTimeField() post_data = fields.DictField() batch_token = fields.StringField() indexes = [ ('channel_id', 'reserved_until'), ('batch_token', ), ]
class JourneyStage(ABCPredictor): collection = 'JourneyStage' manager = JourneyStageManager journey_id = fields.ObjectIdField(db_field='jo') stage_type_id = fields.ObjectIdField( db_field='st') # Will be reference to a JourneyStageType stage_name = fields.StringField(db_field='sn') effort_info = fields.DictField( db_field='ef' ) # Embedded doc with any effort info we're going to track reward_info = fields.DictField( db_field='ri' ) # Embedded doc with any reward info we're going to track start_date = fields.DateTimeField(db_field='sd') end_date = fields.DateTimeField(db_field='ed') last_updated = fields.DateTimeField( db_field='lu' ) # We're probably going to want to know when was the last even from this stage directly last_event = fields.EventIdField( db_field='le') # Keep track of the event itself def check_preconditions(self, event): if hasattr(event, 'stage_id') and event.stage_id: # If 'stage_id' exists on event, and it's not None, that will be a precondition and acceptance rule return str(self.id) == event.stage_id if hasattr(event, 'journeys') and event.journeys: # If a specific set of journeys was passed with the event, the journey for this stage return self.journey_id in event.journeys return True def rule_based_match(self, object): if hasattr(object, 'stage_id') and object.stage_id: # If 'stage_id' exists on event, and it's not None, that will be a precondition and acceptance rule return str(self.id) == object.stage_id return False def process_event(self, event): update_dict = dict(set__last_event=event.data['_id'], set__last_updated=event.datetime_from_id) self.update(**update_dict) def get_journey_stage_type(self): from solariat_bottle.db.journeys.journey_type import JourneyStageType return JourneyStageType.objects.get(self.stage_type_id)
class ContactLabel(AuthDocument): admin_roles = [ADMIN, STAFF, ANALYST] manager = ContactLabelManager title = fields.StringField(db_field='te') created = fields.DateTimeField(db_field='cd', default=datetime.utcnow()) platform = fields.StringField(db_field='pm') status = fields.StringField(db_field='st') users = fields.ListField(fields.StringField()) allow_inheritance = True @classmethod def class_based_access(cls, account): """ Based on the AUTH class we are creating, we might offer some default access to certain groups from the account. By default, permissions should only be given to admin type users. This can be overwritten in specific classes as needed. E.G. messages -> agents ? """ if account is None: return [] return [ default_admin_group(account), default_analyst_group(account), default_reviewer_group(account) ] @property def type_id(self): return 0
class FacetCache(AuthDocument): collection = "FacetCache" hashcode = fields.StringField(db_field='hc', required=True) page_type = fields.StringField(db_field='pe', required=True) account_id = fields.ObjectIdField(db_field='aid', required=True) value = fields.StringField(db_field='ve', required=True) created_at = fields.DateTimeField(db_field='ct', required=True) def is_up_to_date(self): delta = datetime.now() - self.created_at return delta.total_seconds() < get_var( 'MONGO_CACHE_EXPIRATION') # 30 mins @classmethod def upsert_cache_record(cls, hashcode, data, page_type, account_id): now = datetime.now() if 'time_stats' in data: del data['time_stats'] if 'pipelines' in data: del data['pipelines'] cache_records = FacetCache.objects(hashcode=hashcode, account_id=account_id, page_type=page_type) cache_records_num = cache_records.count() if cache_records_num >= 2: raise Exception('Too many cache records') elif cache_records_num == 1: cache = cache_records[0] cache.value = json.dumps(data) cache.created_at = datetime.now() cache.save() else: cache = FacetCache(hashcode=hashcode, value=json.dumps(data), account_id=account_id, page_type=page_type, created_at=now) cache.save() @classmethod def get_cache(cls, params, account_id, page_type): hash_arg = params.copy() # import ipdb; ipdb.set_trace() for field in ['force_recompute', 'range_alias']: if field in hash_arg: del hash_arg[field] hashcode = md5(str(hash_arg)).hexdigest() cache_candidates = FacetCache.objects(hashcode=hashcode, account_id=account_id, page_type=page_type) if not cache_candidates: return hashcode, None elif 1 == len(cache_candidates): return hashcode, cache_candidates[0] else: for cache in cache_candidates: cache.remove() return hashcode, None
class AgentProfile(DynamicProfile): pass cardinalities_lu = fields.DateTimeField() has_indexes = fields.BooleanField(default=False) refresh_rate = timedelta(hours=2) def compute_cardinalities(self): if self.cardinalities_lu: if self.cardinalities_lu + self.refresh_rate >= datetime.utcnow(): return super(AgentProfile, self).compute_cardinalities() if self.cardinalities: self.cardinalities_lu = datetime.utcnow() if not self.has_indexes: self.create_indexes() self.save() def create_indexes(self): coll = self.data_coll # For bigger collections try and create indexes if self.id_field: coll.create_index(self.id_field) for key, value in self.cardinalities.iteritems(): try: coll.create_index(key) except OperationFailure, ex: LOGGER.warning( "Mongo operation failed while trying to create index: %s", ex) if self.created_at_field: coll.create_index(self.created_at_field) self.has_indexes = True
class NPSPost(Post): PROFILE_CLASS = NPSProfile manager = NPSPostManager allow_inheritance = True collection = "NPSPost" case_number = fields.StringField(db_field='cn', required=True) last_modified = fields.DateTimeField(db_field='lm', required=False) case_update_id = fields.StringField(db_field='cd', required=False) type = fields.StringField(db_field='te', required=False) native_id = fields.StringField(db_field='nd', required=True) description = fields.StringField(db_field='dsc') survey_response_name = fields.StringField(db_field='srn') @classmethod def gen_id(cls, account, actor_id, _created, in_reply_to_native_id, parent_event=None): CustomerProfile = account.get_customer_profile_class() actor_num = CustomerProfile.objects.get(id=actor_id).actor_num packed = pack_event_id(actor_num, _created) return packed @property def computed_tags(self): return list( set(self._computed_tags + [str(smt.id) for smt in self.accepted_smart_tags] + self.assigned_tags))
class TaskMessage(Document): ''' Internal Structure representing the integartion data structure with a data stream provider. ''' _created = fields.DateTimeField(db_field='ca', default=datetime.now()) content = fields.StringField(db_field='ct', required=True) type = fields.StringField(db_field='tp', required=True) user = fields.ListField(fields.ReferenceField(User)) manager = TaskMessageManager def add_item(self): ''' Increment counters''' self._update_item(1) def remove_item(self): ''' Decrement counters or remove if empty ''' if self.entry_count >= 2: self._update_item(-1) else: self.delete() def set_datasift_hash(self, datasift_hash): " set atomically datasift hash and update last_sync " return self.objects.coll.find_and_modify( query={'_id': self.id}, update={ '$set': { self.fields['datasift_hash'].db_field: datasift_hash, self.fields['last_sync'].db_field: datetime.now() } }, new=True)
class PostFilter(Document): ''' Internal Structure representing the integartion data structure with a data stream provider. ''' filter_type_id = fields.NumField(db_field='fd', choices=FILTER_TYPE_IDS) # How many entries entry_count = fields.NumField(db_field='et', default=0) # How many more entries can you handle spare_capacity = fields.NumField(db_field='sy', default=POSTFILTER_CAPACITY) datasift_hash = fields.StringField(db_field='dh') last_update = fields.DateTimeField(db_field='lu', default=datetime.now()) last_sync = fields.DateTimeField(db_field='ls') def _update_item(self, n): self.update(inc__entry_count=n, inc__spare_capacity=-n, set__last_update=datetime.now()) def add_item(self): ''' Increment counters''' self._update_item(1) def remove_item(self): ''' Decrement counters or remove if empty ''' if self.entry_count >= 2: self._update_item(-1) else: self.delete() def set_datasift_hash(self, datasift_hash): " set atomically datasift hash and update last_sync " return self.objects.coll.find_and_modify( query={'_id': self.id}, update={ '$set': { self.fields['datasift_hash'].db_field: datasift_hash, self.fields['last_sync'].db_field: datetime.now() } }, new=True)
class BaseProfile(AuthDocument): manager = ProfileManager allow_inheritance = True collection = "BaseProfiles" account_id = fields.ObjectIdField() first_name = fields.StringField() last_name = fields.StringField() age = fields.NumField() sex = fields.StringField() location = fields.StringField() seniority = fields.StringField() assigned_labels = fields.ListField(fields.ObjectIdField()) date_of_birth = fields.StringField() attached_data = fields.DictField() products = fields.ListField(fields.StringField()) actor_num = AutoIncrementField(counter_name='ActorCounter', db_field='ar') created_at = fields.DateTimeField(default=now) linked_profile_ids = fields.ListField(fields.StringField()) indexes = ['actor_num', 'linked_profile_ids'] @property def linked_profiles(self): from solariat_bottle.db.user_profiles.user_profile import UserProfile return UserProfile.objects(id__in=self.linked_profile_ids)[:] def get_profile_of_type(self, typename): if not isinstance(typename, basestring): typename = typename.__name__ for profile in self.linked_profiles: if profile.__class__.__name__ == typename: return profile def add_profile(self, profile): new_id = str(profile.id) if new_id not in self.linked_profile_ids: self.linked_profile_ids.append(new_id) self.update(addToSet__linked_profile_ids=new_id) def get_age(self): # Best guess we can make is by date of birth if present and properly formatted if self.date_of_birth: try: dob = datetime.strptime(self.date_of_birth, AGE_FORMAT) return relativedelta(datetime.now(), dob).years except Exception, ex: LOGGER.error(ex) # Next, if actual age is present, use that but also store updated dob if self.age: dob = datetime.now() - relativedelta(years=self.age) self.date_of_birth = dob.strftime(AGE_FORMAT) self.save() return self.age return None
class EventTag(ABCPredictor): indexes = [('account_id', 'is_multi', ), ] display_name = fields.StringField() account_id = fields.ObjectIdField() status = fields.StringField(default="Active") description = fields.StringField() created = fields.DateTimeField() channels = fields.ListField(fields.ObjectIdField()) manager = EventTagManager default_threshold = 0.49 @property def inclusion_threshold(self): return self.default_threshold def save(self): self.packed_clf = self.clf.packed_model super(EventTag, self).save() def match(self, event): assert isinstance(event, Event), "EventTag expects Event objects" if self.score(event) > self.inclusion_threshold: return True return False def score(self, event): assert isinstance(event, Event), "EventTag expects Event objects" return super(EventTag, self).score(event) def accept(self, event): assert isinstance(event, Event), "EventTag expects Event objects" return super(EventTag, self).accept(event) def reject(self, event): assert isinstance(event, Event), "EventTag expects Event objects" return super(EventTag, self).reject(event) def check_preconditions(self, event): if self.precondition: return eval(self.precondition) return self.feature_extractor.check_preconditions(event, self.features_metadata) def rule_based_match(self, event): if self.acceptance_rule: return eval(self.acceptance_rule) return False def to_dict(self, fields_to_show=None): result_dict = super(EventTag, self).to_dict() result_dict.pop('counter') result_dict.pop('packed_clf') result_dict['channels'] = [str(c) for c in result_dict['channels']] return result_dict
class WidgetModel(Document): """ A WidgetModel is a abstract widget that can be instantiated to ConcreteWidget and used in corresponding typed dashboard """ title = fields.StringField(required=True, unique=True) description = fields.StringField() settings = fields.DictField() created = fields.DateTimeField(default=datetime.now)
class Gallery(Document): """ A gallery is dashboard_type specific, will contain collection of predifined widget models """ dashboard_type = fields.ReferenceField(DashboardType, required=True) widget_models = fields.ListField(fields.ReferenceField(WidgetModel)) created = fields.DateTimeField(default=datetime.now) def to_dict(self): rv = super(Gallery, self).to_dict() rv['display_name'] = self.dashboard_type.display_name rv['type'] = self.dashboard_type.type return rv
class Funnel(AuthDocument): """ """ name = fields.StringField(required=True, unique=True) description = fields.StringField() journey_type = fields.ObjectIdField() steps = fields.ListField(fields.ObjectIdField(), required=True) owner = fields.ReferenceField(User) created = fields.DateTimeField(default=datetime.now) def to_dict(self, fields_to_show=None): rv = super(Funnel, self).to_dict() rv['steps'] = map(str, self.steps) return rv
class PredictorModel(Document): collection = 'PredictorModel' allow_inheritance = True version = fields.NumField() predictor = fields.ReferenceField('BasePredictor') parent = fields.ObjectIdField() weight = fields.NumField() display_name = fields.StringField() description = fields.StringField() # is_active = fields.BooleanField(default=False) task_data = fields.EmbeddedDocumentField(TaskData) last_run = fields.DateTimeField() context_features = fields.ListField(fields.DictField()) action_features = fields.ListField(fields.DictField()) train_data_percentage = fields.NumField(default=80) n_rows = fields.NumField() min_samples_thresould = fields.NumField(default=1) from_dt = fields.DateTimeField() to_dt = fields.DateTimeField() def score(self, *args, **kwargs): pass def feedback(self, *args, **kwargs): pass def search(self, *args, **kwargs): pass def to_json(self, *args, **kwargs): from solariat_bottle.db.predictors.base_predictor import PredictorConfigurationConversion data = super(PredictorModel, self).to_json(*args, **kwargs) data = PredictorConfigurationConversion.python_to_json(data) return data
class BaseScore(AuthDocument): created = fields.DateTimeField(default=now) matching_engine = fields.ObjectIdField() model_id = fields.ObjectIdField(null=True) counter = fields.NumField(default=1) cumulative_latency = fields.NumField(required=True) indexes = [ ('matching_engine', 'created'), ] @property def latency(self): return 1.0 * self.cumulative_latency / (self.counter or 1)
class DashboardWidget(Document): ''' Internal Structure representing the integartion data structure with a data stream provider. ''' created = fields.DateTimeField(db_field='c', default=datetime.now) settings = fields.DictField(db_field='s') order = fields.NumField(db_field='o') title = fields.StringField(db_field='t', required=True) user = fields.ReferenceField(User, db_field='u') dashboard_id = fields.ObjectIdField(required=True) manager = DashboardWidgetManager def to_dict(self): base_dict = dict(title=self.title, order=self.order, id=str(self.id), dashboard_id=str(self.dashboard_id)) base_dict.update(self.settings) return base_dict def copy_to(self, dashboard): new_widget_data = { 'title': self.title, 'user': dashboard.owner, 'dashboard_id': dashboard.id, } new_widget_data.update(self.settings) widget = DashboardWidget.objects.create_by_user(**new_widget_data) return widget def delete(self): dashboard = Dashboard.objects.get_by_user(self.user, id=self.dashboard_id) dashboard._remove_widget(self) super(DashboardWidget, self).delete() def __repr__(self): return "<DashboardWidget: %s; id: %s>" % (self.title, self.id)
class DashboardType(Document): collection = 'DashboardType' manager = DashboardTypeManager type = fields.StringField(required=True, unique=True) display_name = fields.StringField(required=True, unique=True) owner = fields.ReferenceField(User) created = fields.DateTimeField(default=datetime.now) def __repr__(self): return "<DashboardType: %s; id: %s>" % (self.display_name, self.id) def to_dict(self): rv = super(DashboardType, self).to_dict() if self.owner: rv['owner_name'] = '%s %s' % (self.owner.first_name or '', self.owner.last_name or '') rv['email'] = self.owner.email else: rv['owner_name'] = '' rv['email'] = '' return rv
class BaseFeedback(AuthDocument): created = fields.DateTimeField(default=now) action = fields.DictField() context = fields.DictField() matching_engine = fields.ObjectIdField() model_id = fields.ObjectIdField(null=True) reward = fields.NumField() # predicted score est_reward = fields.NumField() context_vector = fields.DictField() action_vector = fields.DictField() # scoring latency in ms score_runtime = fields.NumField() # time taken in millisecond to compute score # scoring error % score_diff = fields.NumField() # (reward - score) / reward indexes = [('matching_engine', 'created'), ]
class TaskData(SonDocument): updated_at = fields.DateTimeField() total = fields.NumField() done = fields.NumField() @property def status(self): return 'training' if 0 < self.done < self.total else 'idle' @property def progress(self): return int(100 * (self.done or 0.0) / (self.total or 1.0)) def to_dict(self, fields_to_show=None): task = self json_data = super(TaskData, self).to_dict(fields_to_show) json_data.update( progress=self.progress, status=task.status, updated_at=task.updated_at and str(task.updated_at) ) return json_data
class Dashboard(AuthDocument): collection = 'Dashboard' manager = DashboardManager type_id = fields.ObjectIdField(required=True) title = fields.StringField(required=True) description = fields.StringField() owner = fields.ReferenceField(User) author = fields.ReferenceField(User) widgets = fields.ListField(fields.ObjectIdField()) shared_to = fields.ListField(fields.ObjectIdField()) filters = fields.DictField() created = fields.DateTimeField(default=datetime.now) admin_roles = {STAFF, ADMIN, REVIEWER, ANALYST} def to_dict(self, fields_to_show=None): rv = super(Dashboard, self).to_dict() rv['widgets'] = map(str, self.widgets) rv['shared_to'] = map(str, self.shared_to) rv['owner_name'] = '%s %s' % (self.owner.first_name or '', self.owner.last_name or '') rv['author_name'] = '%s %s' % (self.author.first_name or '', self.author.last_name or '') rv['owner_email'] = self.owner.email rv['author_email'] = self.author.email rv['account_id'] = str(self.owner.account.id) rv['type'] = DashboardType.objects.get(self.type_id).type return rv def __repr__(self): return "<Dashboard: %s; id: %s>" % (self.title, self.id) def _add_widget(self, widget): """ """ self.widgets.append(widget.id) self.save() def _remove_widget(self, widget): """ widget is not automatically deleted. To delete, use `.delete_widget()` instead. `widget.dashboard_id` will still point to this dashboard. """ self.widgets.remove(widget.id) self.save() def delete_widget(self, widget): if isinstance(widget, (basestring, fields.ObjectId)): widget = DashboardWidget.objects.get(widget) widget.delete() def delete(self): for widget_id in self.widgets: self.delete_widget(widget_id) super(Dashboard, self).delete_by_user(self.owner) def copy_to(self, user, title=None, description=None): dashboard_data = { 'type_id': self.type_id, 'title': title or self.title, 'description': description or self.description, 'author': self.owner, 'owner': user, 'widgets': [], 'shared_to': [], 'filters': self.filters, } # FIX: create_by_user is having role error dashboard = Dashboard.objects.create_by_user(user, **dashboard_data) #dashboard = Dashboard.objects.create(**dashboard_data) for widget_id in self.widgets: widget = DashboardWidget.objects.get(widget_id) widget.copy_to(dashboard) return dashboard
class BrokenQueueMessage(QueueMessage): manager = BrokenQueueMessageManager persisted_at = fields.DateTimeField(default=datetime.utcnow)
class TimeMark(SonDocument): PERIODIC_DATE_FORMAT = '%m-%d' STATIC_DATE_FORMAT = '%Y-%m-%d' TIME_FORMAT = '%H:%M:%S' DAY_OF_WEEK = 0 PERIODIC_DATE = 1 STATIC_DATE = 2 TYPE_NAME_MAP = { DAY_OF_WEEK: 'dayofweek', PERIODIC_DATE: 'periodicdate', STATIC_DATE: 'staticdate'} TYPE_CLASS_MAP = property(lambda self: { self.DAY_OF_WEEK: DayOfWeek, self.PERIODIC_DATE: PeriodicDate, self.STATIC_DATE: StaticDate}) mark_type = fields.NumField(choices=(DAY_OF_WEEK, PERIODIC_DATE, STATIC_DATE), db_field='t') value = fields.StringField() _from_time = fields.DateTimeField(db_field='a', default=None, null=True) _to_time = fields.DateTimeField(db_field='b', default=None, null=True) @property def from_time(self): return self._from_time and self._from_time.time() or time() @property def from_time_fmt(self): return self._from_time and self.from_time.strftime(self.TIME_FORMAT) @property def to_time(self): return self._to_time and self._to_time.time() or time(23, 59, 59, 999999) @property def to_time_fmt(self): return self._to_time and self.to_time.strftime(self.TIME_FORMAT) @property def mark_type_str(self): return self.TYPE_NAME_MAP[self.mark_type] def __init__(self, data=None, **kw): if self.__class__ != TimeMark and not ('mark_type' in kw or data and 't' in data): m = inverse(self.TYPE_CLASS_MAP) kw.update(mark_type=m[self.__class__]) super(TimeMark, self).__init__(data=data, **kw) class_ = self.TYPE_CLASS_MAP[self.data['t']] if class_ != self.__class__: self.__class__ = class_ class_.validate(self) def __hash__(self): return (self.mark_type, self.value, self.from_time_fmt, self.to_time_fmt) def __str__(self): return u"<%s %s %s %s>" % (self.__class__.__name__, self.value, self.from_time_fmt, self.to_time_fmt) def __eq__(self, other): return self.__hash__() == other.__hash__() def __lt__(self, other): raise TypeError("%s is not comparable type" % self.__class__.__name__) def to_json(self, fields_to_show=None): result = { "type": self.mark_type_str, "from": self.from_time_fmt, "to": self.to_time_fmt, "value": self.value } return result @classmethod def from_json(cls, data): m = inverse(cls.TYPE_NAME_MAP) type_ = m[data['type']] from_time = None to_time = None if data['from']: from_time = datetime.strptime(data['from'], cls.TIME_FORMAT) if data['to']: to_time = datetime.strptime(data['to'], cls.TIME_FORMAT) return cls( mark_type=type_, value=data['value'], _from_time=from_time, _to_time=to_time ) def time_points(self): return self.from_time, self.to_time def validate(self): assert self.from_time < self.to_time, "from_time must be less than to_time"
class JobStatus(ArchivingAuthDocument): STATUSES = PENDING, RUNNING, ABANDONED, SUCCESSFUL, FAILED, \ RESUBMITTED, SLEEPING, TERMINATED = \ 'Pending', 'Running', 'Abandoned', 'Successful', 'Failed', \ 'Resubmitted', 'Sleeping', 'Terminated' RUNNABLE_STATUSES = PENDING, SLEEPING collection = 'jobs' account = fields.ObjectIdField(null=True) topic = fields.StringField() name = fields.StringField() args = fields.PickledField() kwargs = fields.PickledField() metadata = fields.DictField(null=True) created_at = fields.DateTimeField() started_date = fields.DateTimeField() completion_date = fields.DateTimeField() status = fields.StringField(choices=STATUSES) state = fields.DictField() last_activity = fields.DateTimeField() awake_at = fields.DateTimeField(null=True) @property def git_commit(self): return (self.metadata or {}).get('git_commit') @property def resubmission_info(self): return (self.metadata or {}).get('resubmitted') def abandon(self): if self.status == self.PENDING: self.status = self.ABANDONED res = self.objects.coll.update( { self.F.id: self.id, self.F.status: self.PENDING }, {"$set": { self.F.status: self.ABANDONED }}) if isinstance(res, dict) and res.get('nModified') == 1: return True def resume(self): if self.status != self.FAILED: raise RuntimeError("Job can not be resumed in '{}' state.".format( self.status)) from solariat_bottle.jobs.manager import manager job = manager.registry.get(self.name) res = job.submit(self.topic, self.name, self.args, self.kwargs, self.metadata) # updating old job meta = self.metadata or {} meta.update(resubmitted={ 'new_id': res.job_instance.id, 'result': str(res.submission_result) }) self.update(status=JobStatus.RESUBMITTED, metadata=meta) # updating new job meta = res.job_instance.metadata or {} meta.update(resubmitted={'old_id': self.id}) res.job_instance.update(metadata=meta) return [self, res.job_instance] def can_edit(self, user_or_group, admin_roles=None): if admin_roles is None: admin_roles = self.admin_roles account_check = user_or_group.is_staff or (user_or_group.is_admin and user_or_group.account.id == self.account) edit_check = (bool( set(admin_roles).intersection(set(user_or_group.user_roles))) or (hasattr(user_or_group, 'is_superuser') and user_or_group.is_superuser)) return account_check and edit_check @property def wait_time(self): if self.started_date and self.created_at: return (utc(self.started_date or now()) - utc(self.created_at)).total_seconds() @property def execution_time(self): if self.completion_date and self.started_date: now_ = now() return (utc(self.completion_date or now_) - utc(self.started_date or now_)).total_seconds()
class CustomerJourney(AuthDocument, EventSequenceStatsMixin): FEAT_TYPE = 'type' FEAT_LABEL = 'label' FEAT_EXPR = 'field_expr' FEAT_NAME = 'name' collection = "CustomerJourney" manager = CustomerJourneyManager stage_name = fields.StringField( db_field='fs') # stage_name of current_stage # Dict in the form: # <strategy_type> : <list of index__stage_name>. strategy_type can be for now (default, platform, event_type) stage_sequences = fields.DictField(db_field='sseq') # Dict in the form # index__stage_name: {actual attributes computed for this specific stage} stage_information = fields.DictField(db_field='si') customer_id = fields.BaseField( db_field='ci') # dynamic profiles may use custom id type customer_name = fields.StringField( db_field='cn') # Just for quick access w/o extra db call agent_ids = fields.ListField(fields.ObjectIdField(), db_field='ag') agent_names = fields.ListField( fields.StringField(), db_field='ans') # Just for quick access w/o extra db calls journey_tags = fields.ListField(fields.ObjectIdField(), db_field='jts') channels = fields.ListField(fields.ObjectIdField(), db_field='chls') last_updated = fields.DateTimeField(db_field='lu') # time spent by events in each stage-eventtype status node_sequence = fields.ListField(fields.DictField(), db_field='nds') node_sequence_agr = fields.ListField(fields.StringField(), db_field='ndsn') # time spent by events in each stage-eventtype status journey_attributes_schema = fields.ListField(fields.DictField(), db_field='jas') first_event_date = fields.DateTimeField(db_field='fed') last_event_date = fields.DateTimeField(db_field='led') indexes = [('journey_type_id', 'journey_tags'), ('journey_attributes', ), ('journey_type_id', 'channels'), ('customer_id', ), ('agent_ids', )] parsers_cache = dict() @classmethod def to_mongo(cls, data, fill_defaults=True): """ Same as super method, except parser.evaluate is skipped (would be called in process_event) """ return super(CustomerJourney, cls).to_mongo(data, fill_defaults=fill_defaults, evaluate=False) @classmethod def metric_label(cls, metric, param, value): # from solariat_bottle.db.predictors.customer_segment import CustomerSegment if param == 'status': value = JourneyStageType.STATUS_TEXT_MAP[value] if param == 'journey_type_id': value = JourneyType.objects.get(value).display_name #value = value[0] if type(value) in [list, tuple] and value else value if value is not None else 'N/A' if value is None: value = 'N/A' return str(value) def ui_repr(self): base_repr = "Status: %s; Start date: %s; End date: %s;" % ( self.status, self.first_event_date, self.last_event_date) if self.customer_name: base_repr += " Customer: %s;" % self.customer_name if self.agent_names: base_repr += " Agents: %s;" % self.agent_names return base_repr def to_dict(self, *args, **kwargs): # from solariat_bottle.db.predictors.customer_segment import CustomerSegment base_dict = super(CustomerJourney, self).to_dict() base_dict['agents'] = map(str, self.agents) base_dict['channels'] = map(str, self.channels) base_dict['smart_tags'] = map(str, self.smart_tags) base_dict['journey_tags'] = map(str, self.journey_tags) base_dict['status'] = JourneyStageType.STATUS_TEXT_MAP[self.status] base_dict['string_repr'] = self.ui_repr() base_dict['journey_attributes'] = self.journey_attributes return base_dict def handle_add_tag(self, tag_id): tag_id = ObjectId(tag_id) self.update(addToSet__smart_tags=tag_id) def handle_remove_tag(self, tag_id): tag_id = ObjectId(tag_id) self.update(pull__smart_tags=tag_id) def apply_schema(self, expression, context): hash_key = str(expression) + '__'.join(context) if hash_key in CustomerJourney.parsers_cache: parser = CustomerJourney.parsers_cache[hash_key] else: parser = BaseParser(expression, context.keys()) CustomerJourney.parsers_cache[hash_key] = parser try: value = parser.evaluate(context) except TypeError: value = None return value def process_event(self, event, customer, agent, journey_stage_type): self._current_event = event received_event_from_past = False created_at = utc(event.created_at) last_updated = utc(self.last_updated) if self.last_updated else None if last_updated and created_at < last_updated: # log.error("=========RECEIVED EVENT FROM THE PAST %s %s < last updated %s" % ( # event, event.created_at, self.last_updated)) received_event_from_past = True # IMPORTANT: No mongo calls should be done here at all! if agent: if agent.id not in self.agent_ids: self.agent_ids.append(agent.id) # TODO: This needs to be enforced on profile dynamic classes as a separate specific # column (can be optional) self.agent_names.append(str(agent)) # TODO: Same as for agent profile, this needs to be set on dynamic class level self.customer_name = str(customer) if event.channels[0] not in self.channels: self.channels.append(event.channels[0]) if not received_event_from_past: if journey_stage_type: self.status = journey_stage_type.status self.last_event_date = event.created_at self.last_updated = event.created_at # TODO: This whole strategy switch will need to be changed to be defined somehow on journey level # TODO: ISSSUE for the last stage the information is not copied. Will need to do this on journey closure. for strategy in [ STRATEGY_DEFAULT, STRATEGY_PLATFORM, STRATEGY_EVENT_TYPE ]: self.check_for_stage_transition(strategy, event, journey_stage_type) schema_computed_attributes = dict() # All of these need to be returned directly from customer data (no extra mongo calls!) expression_context = dict(agents=self.agents, customer_profile=self.customer_profile, current_event=event, event_sequence=self.event_sequence, current_stage=self.current_stage, previous_stage=self.previous_stage, stage_sequence=self.stage_sequence) # for k in self.field_names: # expression_context[k] = getattr(self, k) # adding func with @property decorator to context for key in CustomerJourney.get_properties(): expression_context[key] = getattr(self, key) for schema_entry in self.journey_attributes_schema: expression = schema_entry[self.FEAT_EXPR] f_name = schema_entry[self.FEAT_NAME] schema_computed_attributes[f_name] = self.apply_schema( expression, expression_context) expression_context[f_name] = schema_computed_attributes[f_name] self.journey_attributes = schema_computed_attributes if self.status in [ JourneyStageType.COMPLETED, JourneyStageType.TERMINATED ]: self.node_sequence_agr = [] for i, item in enumerate(self.node_sequence): key, value = item.items()[0] self.node_sequence_agr.append(key) @classmethod def get_properties(cls): """ returns all list of member funcs decorated with @property """ from copy import deepcopy base = deepcopy(cls.field_names) base = [ field for field in base if field not in ('is_archived', '_t', 'acl', 'match_expression', 'journey_type_id', 'display_name', 'account_id', 'id', 'available_stages') ] base.extend([ name for name, value in vars(cls).items() if isinstance(value, property) ]) return base @property def CONSTANT_DATE_NOW(self): # For being picked up by context from datetime import datetime return datetime.now() @property def CONSTANT_ONE_DAYS(self): from datetime import timedelta return timedelta(hours=24) @property def CONSTANT_ONE_HOUR(self): from datetime import timedelta return timedelta(hours=1) def check_for_stage_transition(self, strategy, event, journey_stage_type): current_stage = self.get_current_stage(strategy) if strategy == STRATEGY_DEFAULT: new_stage = journey_stage_type.display_name if journey_stage_type else current_stage elif strategy == STRATEGY_EVENT_TYPE: new_stage = journey_stage_type.display_name + ':' + str( event.event_type) if journey_stage_type else current_stage elif strategy == STRATEGY_PLATFORM: new_stage = journey_stage_type.display_name + ':' + event.platform if journey_stage_type else current_stage if new_stage != current_stage: stage_index = self.get_current_index(strategy) new_stage_value = STAGE_INDEX_SEPARATOR.join( [new_stage, str(stage_index + 1)]) if current_stage is not None: full_stage_name = STAGE_INDEX_SEPARATOR.join( [current_stage, str(stage_index)]) self.stage_information[ full_stage_name] = self.compute_stage_information(strategy) self.stage_sequences[strategy] = self.stage_sequences.get( strategy, []) + [new_stage_value] if strategy == STRATEGY_DEFAULT: self.stage_name = journey_stage_type.display_name self.stage_sequence_names.append(new_stage) if strategy == STRATEGY_EVENT_TYPE: if current_stage is None and new_stage is None: return # TODO: This is still kind of hard coded for MPC if new_stage != current_stage: self.node_sequence.append({new_stage: 1}) else: self.node_sequence[-1][new_stage] += 1 def compute_stage_information(self, strategy): info = dict() for key, val in self.journey_attributes.iteritems(): info[key] = val info['end_date'] = self.last_event_date if len(self.stage_sequences.get(strategy, [])) <= 1: info['start_date'] = self.first_event_date else: info['start_date'] = self.stage_information[ self.stage_sequences[strategy][-2]]['end_date'] return info def close_journey(self): for strategy in [ STRATEGY_DEFAULT, STRATEGY_PLATFORM, STRATEGY_EVENT_TYPE ]: current_stage = self.get_current_stage(strategy) stage_index = self.get_current_index(strategy) if current_stage is not None: full_stage_name = STAGE_INDEX_SEPARATOR.join( [current_stage, str(stage_index)]) self.stage_information[ full_stage_name] = self.compute_stage_information(strategy) self.save() def get_current_stage(self, strategy_type): if not self.stage_sequences.get(strategy_type): return None else: return self.stage_sequences.get(strategy_type)[-1].split( STAGE_INDEX_SEPARATOR)[0] def get_current_index(self, strategy_type): if not self.stage_sequences.get(strategy_type): return -1 else: return int( self.stage_sequences.get(strategy_type)[-1].split( STAGE_INDEX_SEPARATOR)[1]) def stage_sequence_by_strategy(self, strategy): return [ val.split(STAGE_INDEX_SEPARATOR)[0] for val in self.stage_sequences[strategy] ] def __get_agents(self): if hasattr(self, '_agents'): return self._agents else: self._agents = self.account.get_agent_profile_class().objects.find( id__in=self.agent_ids)[:] return self._agents def __set_agents(self, agents): self._agents = agents agents = property(__get_agents, __set_agents) def __get_customer_profile(self): if hasattr(self, '_customer_profile'): return self._customer_profile else: self._customer_profile = self.account.get_customer_profile_class( ).objects.get(self.customer_id) return self._customer_profile def __set_customer_profile(self, customer_profile): self._customer_profile = customer_profile customer_profile = property(__get_customer_profile, __set_customer_profile) def __get_current_event(self): if hasattr(self, '_current_event'): return self._current_event else: self._current_event = self.event_sequence[ -1] if self.event_sequence else None return self._current_event def __set_current_event(self, event): self._current_event = event current_event = property(__get_current_event, __set_current_event) def __get_event_sequence(self): if hasattr(self, '_event_sequence'): return self._event_sequence else: from solariat_bottle.db.account import Account account = Account.objects.get(self.account_id) CustomerProfile = account.get_customer_profile_class() try: customer = CustomerProfile.objects.get(self.customer_id) except CustomerProfile.DoesNotExist: self._event_sequence = [] return self._event_sequence if self.first_event_date and self.last_event_date: events = Event.objects.events_for_actor( self.first_event_date, self.last_event_date, customer.actor_num)[:] self._event_sequence = events return self._event_sequence # event_type_ids = [x.event_type for x in events] # event_types = EventType.objects(id__in=event_type_ids)[:] # event_type_map = {str(x.id): x.name for x in event_types} # return [event_type_map[x.event_type] for x in events] self._event_sequence = [] return self._event_sequence def __set_event_sequence(self, event_sequence): self._event_sequence = event_sequence event_sequence = property(__get_event_sequence, __set_event_sequence) @property def current_stage(self): if len(self.stage_sequences.get(STRATEGY_DEFAULT, [])) == 0: return None else: last_stage = self.stage_sequences[STRATEGY_DEFAULT][-1].split( STAGE_INDEX_SEPARATOR)[0] return last_stage @property def nps(self): nps1 = self.journey_attributes.get('nps') event = self.current_event from solariat_bottle.db.post.nps import NPSOutcome if isinstance(event, NPSOutcome): nps2 = self.current_event.score else: nps2 = None return max(nps1, nps2) @staticmethod def nps_value_to_label(value): if value is None: return 'n/a' elif 0 <= value <= 6: return 'detractor' elif value in (7, 8): return 'passive' elif value in (9, 10): return 'promoter' else: raise Exception("invalid nps value (%r given)" % value) @property def nps_category(self): # from solariat_bottle.views.facets import nps_value_to_label if self.nps == 'N/A': return 'N/A' else: return self.nps_value_to_label(self.nps) @property def previous_stage(self): if self.current_stage is None: return None if len(self.stage_sequences.get(STRATEGY_DEFAULT, [])) <= 1: return None else: last_stage = self.stage_sequences[STRATEGY_DEFAULT][-2].split( STAGE_INDEX_SEPARATOR)[0] return last_stage @property def stage_sequence(self): if len(self.stage_sequences.get(STRATEGY_DEFAULT, [])) == 0: return [] else: return [ val.split(STAGE_INDEX_SEPARATOR)[0] for val in self.stage_sequences[STRATEGY_DEFAULT] ] @property def first_event(self): event_sequence = self.event_sequence if event_sequence: return event_sequence[0] else: return None @property def is_abandoned(self): if self.status == JourneyStageType.TERMINATED: return 1 else: return 0 @property def days(self): if self.first_event_date and self.last_event_date: return (utc(self.last_event_date) - utc(self.first_event_date)).days else: return None