class Questions(me.Document): meta = {'collection': 'questions'} user = me.StringField(required=True) text = me.StringField(required=True) date = me.DateTimeField(required=True)
class User(mongoengine.EmbeddedDocument): username = mongoengine.StringField() user_id = mongoengine.StringField()
class User(me.Document): # The fields needed to display a user's name and profile picture. CORE_FIELDS = ['first_name', 'last_name', 'fbid', 'email'] class JoinSource(object): FACEBOOK = 1 EMAIL = 2 class UserCreationError(Exception): pass meta = { 'indexes': [ 'fb_access_token', 'fbid', # TODO(mack): need to create the 'api_key' index on prod 'api_key', # Allow users with email=None, but non-None emails must be unique { 'fields': ['email'], 'unique': True, 'sparse': True, }, 'referrer_id', ], } # Randomly generated ID used to access some subset of user's information # without going through any ACL. Used for e.g. sharing schedules with non # flow users. # # e.g. A8RLLZTMX secret_id = me.StringField() # TODO(mack): join_date should be encapsulate in _id, but store it # for now, just in case; can remove it when sure that info is in _id join_date = me.DateTimeField(required=True) join_source = me.IntField(required=True, choices=[JoinSource.FACEBOOK, JoinSource.EMAIL]) referrer_id = me.ObjectIdField(required=False) # eg. Mack first_name = me.StringField(required=True) middle_name = me.StringField() # eg. Duan last_name = me.StringField(required=True) # TODO(mack): check if facebook always returns gender field gender = me.StringField(choices=['male', 'female']) # eg. 1647810326 fbid = me.StringField() # http://stackoverflow.com/questions/4408945/what-is-the-length-of-the-access-token-in-facebook-oauth2 fb_access_token = me.StringField(max_length=255) fb_access_token_expiry_date = me.DateTimeField() # The token expired due to de-auth, logging out, etc (ie. not time expired) fb_access_token_invalid = me.BooleanField(default=False) email = me.EmailField() password = me.StringField() # eg. list of user objectids, could be friends from sources besides # facebook friend_ids = me.ListField(me.ObjectIdField()) # eg. list of fbids of friends from facebook, not necessarily all of whom # use the site friend_fbids = me.ListField(me.StringField()) birth_date = me.DateTimeField() last_visited = me.DateTimeField() # TODO(mack): consider using SequenceField() num_visits = me.IntField(min_value=0, default=0) # The last time the user visited the onboarding page last_show_onboarding = me.DateTimeField() # The last time the user was shown the import schedule view last_show_import_schedule = me.DateTimeField() # eg. mduan or 20345619 ? student_id = me.StringField() # eg. university_of_waterloo ? school_id = me.StringField() # eg. software_engineering ? # TODO(mack): should store program_id, not program_name # program_id = me.StringField() program_name = me.StringField() # List of UserCourse.id's course_history = me.ListField(me.ObjectIdField()) # TODO(mack): figure out why last_term_id was commented out in # a prior diff: #260f174 # Deprecated last_term_id = me.StringField() # Deprecated last_program_year_id = me.StringField() # Track the number of times the user has invited friends # (So we can award points if they have) num_invites = me.IntField(min_value=0, default=0) # The number of points this user has. Point are awarded for a number of # actions such as reviewing courses, inviting friends. This is a cached # point total. It will be calculated once a day with aggregator.py num_points = me.IntField(min_value=0, default=0) is_admin = me.BooleanField(default=False) # TODO(mack): refactor this into something maintainable sent_exam_schedule_notifier_email = me.BooleanField(default=False) sent_velocity_demo_notifier_email = me.BooleanField(default=False) sent_raffle_notifier_email = me.BooleanField(default=False) sent_raffle_end_notifier_email = me.BooleanField(default=False) sent_schedule_sharing_notifier_email = me.BooleanField(default=False) sent_course_enrollment_feb_8_email = me.BooleanField(default=False) sent_referral_contest_email = me.BooleanField(default=False) sent_referral_contest_end_email = me.BooleanField(default=False) sent_welcome_email = me.BooleanField(default=False) email_unsubscribed = me.BooleanField(default=False) # Note: Backfilled on night of Nov. 29th, 2012 transcripts_imported = me.IntField(min_value=0, default=0) schedules_imported = me.IntField(min_value=0, default=0) last_bad_schedule_paste = me.StringField() last_good_schedule_paste = me.StringField() last_bad_schedule_paste_date = me.DateTimeField() last_good_schedule_paste_date = me.DateTimeField() # Whether this user imported a schedule when it was still broken and we # should email them to apologize schedule_sorry = me.BooleanField(default=False) # API key that grants user to login_required APIs api_key = me.StringField() last_prompted_for_review = me.DateTimeField(default=datetime.datetime.min) voted_course_review_ids = me.ListField(me.StringField()) voted_prof_review_ids = me.ListField(me.StringField()) # Scholarships where a user has clicked: "Remove from profile" closed_scholarship_ids = me.ListField(me.StringField()) @property def name(self): return '%s %s' % (self.first_name, self.last_name) def save(self, *args, **kwargs): # TODO(mack): If _changed_fields attribute does not exist, it mean # document has been saved yet. Just need to verify. In this case, # we could just check if id has been set first_save = not hasattr(self, '_changed_fields') if first_save: # TODO(Sandy): We're assuming people won't unfriend anyone. # Fix this later? # TODO(mack): this isn't safe against race condition of both # friends signing up at same time #print 'friend_fbids', self.friend_fbids friends = (User.objects(fbid__in=self.friend_fbids).only( 'id', 'friend_ids')) self.friend_ids = [f.id for f in friends] super(User, self).save(*args, **kwargs) if first_save: # TODO(mack): should do this asynchronously # Using update rather than save because it should be more efficient friends.update(add_to_set__friend_ids=self.id) # TODO(mack): think of better way to cache value @property def course_ids(self): if not hasattr(self, '_course_ids'): user_courses = _user_course.UserCourse.objects( id__in=self.course_history).only('course_id') self._course_ids = [uc.course_id for uc in user_courses] return self._course_ids @property def profile_pic_urls(self): if self.fbid is not None: urls = self._get_fb_pic_urls() else: urls = self._get_gravatar_pic_urls() return urls def _get_fb_pic_urls(self): base_pic = "https://graph.facebook.com/%s/picture" % (self.fbid) return { 'default': base_pic, 'large': '%s?type=large' % (base_pic), 'square': '%s?type=square' % (base_pic), } def _get_gravatar_pic_urls(self): # Gravatar API: https://en.gravatar.com/site/implement/images/ # TODO(sandy): Serve our own default image instead of the mystery man email_hash = hashlib.md5(self.email.strip().lower()).hexdigest() base_pic = "https://secure.gravatar.com/avatar/%s?d=mm" % (email_hash) return { 'default': "%s&size=%s" % (base_pic, "50"), 'large': "%s&size=%s" % (base_pic, "190"), 'square': "%s&size=%s" % (base_pic, "50"), } @property def profile_url(self): return '/profile/%s' % self.id @property def absolute_profile_url(self): return '%s%s?admin=1' % (constants.RMC_HOST, self.profile_url) @property def short_program_name(self): if self.program_name: return self.program_name.split(',')[0] return '' @property def has_course_history(self): # TODO(Sandy): Using this to backfill transcripts imported, # remove later if len(self.course_history) == 0: return False for uc in self.get_user_courses(): if not _term.Term.is_shortlist_term(uc.term_id): return True return False @property def has_shortlisted(self): for uc in self.get_user_courses(): if _term.Term.is_shortlist_term(uc.term_id): return True return False @property def has_schedule(self): # TODO(Sandy): Actually this only works assuming users never remove # their schedule and we'll have to do actual queries when 2013_05 comes return self.schedules_imported > 0 @property def should_renew_fb_token(self): # Should renew FB token if it expired or will expire "soon". future_date = datetime.datetime.now() + datetime.timedelta( days=facebook.FB_FORCE_TOKEN_EXPIRATION_DAYS) return (self.fb_access_token_expiry_date is None or self.fb_access_token_expiry_date < future_date or self.fb_access_token_invalid) @property def is_fb_token_expired(self): return (self.fb_access_token_expiry_date is None or self.fb_access_token_expiry_date < datetime.datetime.now() or self.fb_access_token_invalid) @property def is_demo_account(self): return self.fbid == constants.DEMO_ACCOUNT_FBID @property def last_schedule_paste(self): return self.last_good_schedule_paste or self.last_bad_schedule_paste def get_user_courses(self): return _user_course.UserCourse.objects(id__in=self.course_history) @classmethod def cls_mutual_courses_redis_key(cls, user_id_one, user_id_two): if user_id_one < user_id_two: first_id = user_id_one second_id = user_id_two else: first_id = user_id_two second_id = user_id_one return 'mutual_courses:%s:%s' % (first_id, second_id) def mutual_courses_redis_key(self, other_user_id): return User.cls_mutual_courses_redis_key(self.id, other_user_id) def get_mutual_course_ids(self, redis): # fetch mutual friends from redis pipe = redis.pipeline() # Show mutual courses between the viewing user and the friends of the # profile user for friend_id in self.friend_ids: pipe.smembers(self.mutual_courses_redis_key(friend_id)) mutual_course_ids_per_user = pipe.execute() zipped = itertools.izip(self.friend_ids, mutual_course_ids_per_user) mutual_course_ids_by_friend = {} for friend_id, mutual_course_ids in zipped: mutual_course_ids_by_friend[friend_id] = mutual_course_ids return mutual_course_ids_by_friend def cache_mutual_course_ids(self, redis): friends = User.objects(id__in=self.friend_ids).only('course_history') friend_map = {} for friend in friends: friend_map[friend.id] = friend my_course_ids = set(self.course_ids) for friend in friends: mutual_course_ids = my_course_ids.intersection(friend.course_ids) if mutual_course_ids: redis_key = self.mutual_courses_redis_key(friend.id) redis.sadd(redis_key, *list(mutual_course_ids)) def remove_mutual_course_ids(self, redis): pipe = redis.pipeline() for friend_id in self.friend_ids: pipe.delete(self.mutual_courses_redis_key(friend_id)) return pipe.execute() def get_latest_program_year_id(self): latest_term_uc = None for uc_dict in self.get_user_courses(): # Ignore untaken courses or shortlisted courses if uc_dict['term_id'] > util.get_current_term_id(): continue if not latest_term_uc: latest_term_uc = uc_dict elif uc_dict['term_id'] > latest_term_uc['term_id']: latest_term_uc = uc_dict if latest_term_uc: return latest_term_uc['program_year_id'] return None def get_friends(self): """Gets basic info for each of this user's friends.""" return User.objects(id__in=self.friend_ids).only( *(User.CORE_FIELDS + ['id', 'num_points', 'num_invites', 'program_name'])) def rated_review(self, review_id, review_type): if review_type == 'course': return review_id in self.voted_course_review_ids else: return review_id in self.voted_prof_review_ids def to_dict(self, extended=True, include_course_ids=False): user_dict = { 'id': self.id, 'fbid': self.fbid, 'first_name': self.first_name, 'last_name': self.last_name, 'name': self.name, 'profile_pic_urls': self.profile_pic_urls, 'program_name': self.short_program_name, 'num_invites': self.num_invites, 'num_points': self.num_points, } if extended: user_dict.update({ 'friend_ids': self.friend_ids, 'course_history': self.course_history, }) if include_course_ids: user_dict['course_ids'] = self.course_ids return user_dict # TODO(mack): make race condition safe? def delete(self, *args, **kwargs): # Remove this user from the friend lists of all friends friends = User.objects(id__in=self.friend_ids) friends.update(pull__friend_ids=self.id) # Delete all their user course objects _user_course.UserCourse.objects(user_id=self.id).delete() # Delete all their UserScheduleItem objects _user_schedule_item.UserScheduleItem.objects(user_id=self.id).delete() # TODO(mack): delete mutual course information from redis? # should be fine for now since we are removing this user from their # friends' friend_ids, and redis cache will be regenerated daily # from aggregator.py return super(User, self).delete(*args, **kwargs) def to_review_author_dict(self, current_user, reveal_identity): is_current_user = current_user and current_user.id == self.id if reveal_identity: return { 'id': self.id, 'name': 'You' if is_current_user else self.name, 'profile_pic_url': self.profile_pic_urls['square'], } else: return {'program_name': self.short_program_name} def invite_friend(self, redis): self.num_invites += 1 if self.num_invites == 1: self.award_points(_points.PointSource.FIRST_INVITE, redis) def award_points(self, points, redis): self.num_points += points redis.incr('total_points', points) def update_fb_friends(self, fbids): self.friend_fbids = fbids fb_friends = (User.objects(fbid__in=self.friend_fbids).only( 'id', 'friend_ids')) # We have friends from only Facebook right now, so just set it self.friend_ids = [f.id for f in fb_friends] def get_schedule_item_dicts(self, exam_objs=None): """Gets all schedule items for this user starting no later than a year ago. Args: exam_objs: Optional exam objects to convert to UserScheduleItem and add to return list. Returns: a list of UserScheduleItem models as dicts. """ one_year_ago = datetime.datetime.now() - datetime.timedelta(days=365) schedule_item_objs = _user_schedule_item.UserScheduleItem.objects( user_id=self.id, start_date__gte=one_year_ago) dicts = [si.to_dict() for si in schedule_item_objs] if exam_objs: dicts.extend(e.to_schedule_obj().to_dict() for e in exam_objs) return dicts def get_failed_schedule_item_dicts(self): one_year_ago = datetime.datetime.now() - datetime.timedelta(days=365) schedule_item_objs = _user_schedule_item.FailedScheduleItem.objects( user_id=self.id, parsed_date__gte=one_year_ago) return [si.to_dict() for si in schedule_item_objs] def get_all_schedule_items(self): return _user_schedule_item.UserScheduleItem.objects(user_id=self.id) def get_current_term_exams(self, current_term_course_ids=None): if not current_term_course_ids: ucs = (self.get_user_courses().filter( term_id=util.get_current_term_id()).only('course_id')) current_term_course_ids = [uc.course_id for uc in ucs] return _exam.Exam.objects(course_id__in=current_term_course_ids) def get_secret_id(self): # TODO(jlfwong): This is possibly a race condition... if self.secret_id is None: self.secret_id = util.generate_secret_id() self.save() return self.secret_id def add_course(self, course_id, term_id, program_year_id=None): """Creates a UserCourse and adds it to the user's course_history. Idempotent. Returns the resulting UserCourse. """ user_course = _user_course.UserCourse.objects( user_id=self.id, course_id=course_id).first() if user_course is None: if _course.Course.objects.with_id(course_id) is None: # Non-existant course according to our data rmclogger.log_event(rmclogger.LOG_CATEGORY_DATA_MODEL, rmclogger.LOG_EVENT_UNKNOWN_COURSE_ID, course_id) return None user_course = _user_course.UserCourse( user_id=self.id, course_id=course_id, term_id=term_id, program_year_id=program_year_id, ) else: # Record only the latest attempt for duplicate/failed courses if (term_id > user_course.term_id or user_course.term_id == _term.Term.SHORTLIST_TERM_ID): user_course.term_id = term_id user_course.program_year_id = program_year_id user_course.save() if user_course.id not in self.course_history: self.course_history.append(user_course.id) self.save() return user_course # Generate a random api key granting this user to access '/api/' routes def grant_api_key(self): uuid_ = uuid.uuid4() md5 = hashlib.md5() md5.update(str(uuid_)) microsecs = int(time.time() * 1000000) raw_api_key = str(microsecs) + md5.hexdigest() self.api_key = base64.b64encode(raw_api_key) self.save() return self.api_key def next_course_to_review(self): user_courses = _user_course.UserCourse.objects(user_id=self.id) return _user_course.UserCourse.select_course_to_review(user_courses) def should_prompt_review(self): now = datetime.datetime.now() elapsed = min(now - self.last_prompted_for_review, now - self.join_date) return elapsed.days > PROMPT_TO_REVIEW_DELAY_DAYS @staticmethod def auth_user(email, password): """Returns the authenticated user or None.""" user = User.objects(email=email) if not user: return None # TODO(sandy): Since we added a unique index on email, this shouldn't # happen anymore. But keep this around for a bit, in case something # messes up [Apr 8, 2014] if user.count() > 1: logging.error('Multiple email addressed matched: %s' % email) return None user = user.first() # TODO(sandy): Provide more helpful errors for users signed up with fb if (not user.password or not bcrypt.check_password_hash(user.password, password)): return None return user @staticmethod def create_new_user_from_email(first_name, last_name, email, password): if len(password) < PASSWORD_MIN_LENGTH: raise User.UserCreationError( 'Passwords must be at least 8 characters long.') user = User( email=email, first_name=first_name, join_date=datetime.datetime.now(), join_source=User.JoinSource.EMAIL, last_name=last_name, password=bcrypt.generate_password_hash(password, rounds=BCRYPT_ROUNDS), ) try: user.save() except me.base.ValidationError as e: if 'email' in e.errors: raise User.UserCreationError('Oops, that email is invalid.') raise except me.queryset.NotUniqueError as e: raise User.UserCreationError( 'That email is already signed up.' ' (Maybe you already signed up with Facebook?)') return user def __repr__(self): return "<User: %s>" % self.name.encode('utf-8')
class Address(BaseModel): state = mongoengine.StringField() county = mongoengine.StringField() zipcode = mongoengine.StringField() street = mongoengine.StringField() address = mongoengine.StringField()
class PollingSchedule(me.Document): meta = { 'allow_inheritance': True, 'strict': False, } # We use a unique name for easy identification and to avoid running the # same schedule twice. The name is autopopulated during the invocation of # the `clean` method. name = me.StringField(unique=True) # The following fields are defined in celerybeatmongo.models.PeriodicTask. # Here, we define no fields in the base class, and expect subclasses to # either define their fields, or simply use properties. # task = me.StringField(required=True) # args = me.ListField() # kwargs = me.DictField() # Scheduling information. Don't edit them directly, just use the model # methods. default_interval = me.EmbeddedDocumentField( PollingInterval, required=True, default=PollingInterval(every=0)) override_intervals = me.EmbeddedDocumentListField(PollingInterval) # Optional arguments. queue = me.StringField() exchange = me.StringField() routing_key = me.StringField() soft_time_limit = me.IntField() # Used internally by the scheduler. last_run_at = me.DateTimeField() total_run_count = me.IntField(min_value=0) run_immediately = me.BooleanField() def get_name(self): """Construct name based on self.task""" try: return self.task.split('.')[-1] except NotImplementedError: return '%s: No task specified.' % self.__class__.__name__ def clean(self): """Automatically set value of name""" self.name = self.get_name() @property def task(self): """Return task name for this schedule Subclasses should define an attribute, property or field to do this. """ raise NotImplementedError() @property def args(self): """Return task args for this schedule""" return [str(self.id)] @property def kwargs(self): """Return task kwargs for this schedule""" return {} @property def enabled(self): """Whether this task is currently enabled or not""" return bool(self.interval.timedelta) @property def interval(self): """Merge multiple intervals into one Returns a dynamic PollingInterval, with the highest frequency of any override schedule or the default schedule. """ interval = self.default_interval for i in self.override_intervals: if not i.expired(): if not interval.timedelta or i.timedelta < interval.timedelta: interval = i return interval @property def schedule(self): """Return a celery schedule instance This is used internally by celerybeatmongo scheduler """ return celery.schedules.schedule(self.interval.timedelta) @property def expires(self): return None def add_interval(self, interval, ttl=300, name=''): """Add an override schedule to the scheduled task Override schedules must define an interval in seconds, as well as a TTL (time to live), also in seconds. Override schedules cannot be removed, so short TTL's should be used. You can however add a new override schedule again, thus practically extending the time where an override is in effect. Override schedules can only increase, not decrease frequency of the schedule, in relation to that define in the `default_interval`. """ assert isinstance(interval, int) and interval > 0 assert isinstance(ttl, int) and 0 < ttl < 3600 expires = datetime.datetime.now() + datetime.timedelta(seconds=ttl) self.override_intervals.append( PollingInterval(name=name, expires=expires, every=interval)) def cleanup_expired_intervals(self): """Remove override schedules that have expired""" self.override_intervals = [ override for override in self.override_intervals if not override.expired() ] def set_default_interval(self, interval): """Set default interval This is the interval used for this schedule, if there is no active override schedule with a smaller interval. The default interval never expires. To disable a task, simply set `enabled` equal to False. """ self.default_interval = PollingInterval(name='default', every=interval) def __unicode__(self): return "%s %s" % (self.get_name(), self.interval or '(no interval)')
class Video(mongoengine.Document): name = mongoengine.StringField(required=True) theme = mongoengine.StringField(required=True) thumbs_up = mongoengine.IntField(default=0) thumbs_down = mongoengine.IntField(default=0)
class County(BaseModel): state = mongoengine.StringField() county = mongoengine.StringField() meta = {"collection": Model.county}
class MUserStory(mongo.Document): """ Stories read by the user. These are deleted as the mark_read_date for the UserSubscription passes the UserStory date. """ user_id = mongo.IntField() feed_id = mongo.IntField() read_date = mongo.DateTimeField() story_id = mongo.StringField(unique_with=('user_id', 'feed_id')) story_date = mongo.DateTimeField() story = mongo.ReferenceField(MStory) found_story = mongo.GenericReferenceField() meta = { 'collection': 'userstories', 'indexes': [ { 'fields': ('user_id', 'feed_id', 'story_id'), 'unique': True }, ('feed_id', 'story_id'), # Updating stories with new guids ('feed_id', 'story_date'), # Trimming feeds ], 'allow_inheritance': False, 'index_drop_dups': True, } def save(self, *args, **kwargs): self.sync_redis() super(MUserStory, self).save(*args, **kwargs) def delete(self, *args, **kwargs): self.remove_from_redis() super(MUserStory, self).delete(*args, **kwargs) @property def guid_hash(self): return hashlib.sha1(self.story_id).hexdigest() @classmethod def delete_old_stories(cls, feed_id): UNREAD_CUTOFF = datetime.datetime.utcnow() - datetime.timedelta( days=settings.DAYS_OF_UNREAD) cls.objects(feed_id=feed_id, story_date__lte=UNREAD_CUTOFF).delete() @classmethod def delete_marked_as_read_stories(cls, user_id, feed_id, mark_read_date=None): if not mark_read_date: usersub = UserSubscription.objects.get(user__pk=user_id, feed__pk=feed_id) mark_read_date = usersub.mark_read_date # Next line forces only old read stories to be removed, just in case newer stories # come in as unread because they're being shared. mark_read_date = datetime.datetime.utcnow() - datetime.timedelta( days=settings.DAYS_OF_UNREAD) cls.objects(user_id=user_id, feed_id=feed_id, read_date__lte=mark_read_date).delete() @property def story_db_id(self): if self.story: return self.story.id elif self.found_story: if '_ref' in self.found_story: return self.found_story['_ref'].id elif hasattr(self.found_story, 'id'): return self.found_story.id story, found_original = MStory.find_story(self.feed_id, self.story_id) if story: if found_original: self.story = story else: self.found_story = story self.save() return story.id def sync_redis(self, r=None): if not r: r = redis.Redis(connection_pool=settings.REDIS_STORY_POOL) if self.story_db_id: all_read_stories_key = 'RS:%s' % (self.user_id) r.sadd(all_read_stories_key, self.story_db_id) read_story_key = 'RS:%s:%s' % (self.user_id, self.feed_id) r.sadd(read_story_key, self.story_db_id) r.expire(read_story_key, settings.DAYS_OF_UNREAD * 24 * 60 * 60) def remove_from_redis(self): r = redis.Redis(connection_pool=settings.REDIS_STORY_POOL) if self.story_db_id: r.srem('RS:%s' % self.user_id, self.story_db_id) r.srem('RS:%s:%s' % (self.user_id, self.feed_id), self.story_db_id) @classmethod def sync_all_redis(cls): read_stories = cls.objects.all() for read_story in read_stories: read_story.sync_redis()
class Snake: registered_date = mongoengine.DateTimeField(default=datetime.now()) species = mongoengine.StringField(required=True)
class County(me.Document): name = me.StringField(required=True) geometry = me.MultiPolygonField(required=True)
class Jail(me.Document): name = me.StringField(required=True) location = me.PointField(required=True) state = me.StringField()
class Category(me.Document): title = me.StringField(min_length=1, max_length=512) description = me.StringField(min_length=2, max_length=4096) subcategories = me.ListField(me.ReferenceField('self')) parent = me.ReferenceField('self', default=None)
class Artist(mongoengine.Document): first_name = mongoengine.StringField(max_length=50) last_name = mongoengine.StringField(max_length=50)
class User(me.Document): meta = {'collection': 'questions'} user = me.StringField(required=True) admin = me.BooleanField(required=True)
class Material(mongoengine.EmbeddedDocument): """材料属性 """ materialid = mongoengine.StringField(verbose_name="材料ID") amount = mongoengine.IntField(verbose_name="数量", default=0)
class User(me.Document): name = me.StringField() email = me.StringField(nullable=False) #nao permite campo nulo
class User(me.Document): """ status : 'wait for approval' -> wait member approve this profile : 'activate' -> this profile are approve """ meta = {'collection': 'users'} username = me.StringField(required=True, unique=True) password = me.StringField() email = me.EmailField(required=True, unique=True) first_name = me.StringField(max_length=100, required=True) last_name = me.StringField(max_length=100, required=True) display_name = me.StringField(max_length=250, required=True) default_profile = me.StringField(default='pumbaa.coe.psu.ac.th') online_profiles = me.ListField(me.EmbeddedDocumentField(Profile)) status = me.StringField(max_length=100, required=True, default='wait for approval') registration_date = me.DateTimeField(required=True, default=datetime.datetime.now) updated_date = me.DateTimeField(required=True, default=datetime.datetime.now) approvers = me.ListField(me.EmbeddedDocumentField(Approver)) ip_address = me.StringField(max_length=100, required=True, default='0.0.0.0') roles = me.ListField(me.ReferenceField('Role', dbref=True)) def get_display_name(self): if self.display_name is not None: return self.display_name else: return self.username def get_tim_display_name(self): tname = self.get_display_name().split(" ") if len(tname) > 1: return "%s. %s" % (tname[0][0], tname[1]) else: return "%s" % (tname[0]) def set_password(self, password): from pyramid.threadlocal import get_current_request request = get_current_request() self.password = request.secret_manager.get_hash_password(password) def get_profile(self, domain): for profile in self.online_profiles: if profile.domain == domain: return profile def get_role(self, name): for role in self.roles: if role.name == name: return role def get_profile_picture(self, width=50): if self.default_profile == 'pumbaa.coe.psu.ac.th': return None profile = self.get_profile(self.default_profile) if profile.domain == 'facebook.com': if '=' in profile.username: username = profile.username.split('=')[-1] else: username = profile.username return '<img src="https://graph.facebook.com/%s/picture" width="%d"/>' % ( username, width) if profile.domain == 'twitter.com': return '<img src="%s" width="%d"/>' % ( profile.profile_source['photos'][0]['value'], width) if profile.domain == 'accounts.google.com': return '<img src="%s" width="%d"/>' % ( profile.profile_source['photos'][0]['value'], width) return None def get_profile_url(self): if self.default_profile == 'pumbaa.coe.psu.ac.th': return '#' profile = self.get_profile(self.default_profile) if profile.domain == 'facebook.com': return 'https://www.facebook.com/%s' % profile.username if profile.domain == 'twitter.com': return 'https://twitter.com/%s' % profile.username if profile.domain == 'accounts.google.com': return 'https://plus.google.com/%s' % profile.user_id return '#'
class DiscreteMeasurement(Measurement): result = mongoengine.StringField(required=True)
class State(BaseModel): state = mongoengine.StringField() meta = {"collection": Model.state}
class User(mongoengine.Document): first_name = mongoengine.StringField(max_length=128, required=True) last_name = mongoengine.StringField(max_length=128, required=True) email = mongoengine.EmailField(max_length=128, required=True, unique=True)
class Zipcode(BaseModel): state = mongoengine.StringField() county = mongoengine.StringField() zipcode = mongoengine.StringField() meta = {"collection": Model.zipcode}
class Character(me.Document): name = me.StringField(required=True) height = me.StringField(required=True) mass = me.StringField(required=True) birth_year = me.StringField(required=True) gender = me.StringField(required=True) homeworld = me.StringField(required=True) films = me.ListField(me.StringField(), required=True) starships = me.ListField(me.StringField()) vehicles = me.ListField(me.StringField()) planets = me.ListField(me.StringField()) url = me.StringField(required=True) @classmethod def get_all(cls): return cls.objects() @classmethod def get_by_id(cls, id): return cls.objects(id=id) @classmethod def add(cls, name,height,mass,birth_year,gender, url, films, starships, vehicles, homeworld): cls( name=name, height=height, mass=mass, birth_year=birth_year, gender=gender, homeworld=homeworld, url=url, films=[Films.get_by_url(film).to_json() for film in films], starships=[Starships.get_by_url(starship).to_json() for starship in starships], vehicles=[Vehicles.get_by_url(vehicle).to_json() for vehicle in vehicles], planets=[Planets.get_by_url(homeworld).to_json()] ).save()
class DebugPollingSchedule(PollingSchedule): task = 'mist.api.poller.tasks.debug' value = me.StringField()
class Product(me.Document): name = me.StringField(required=True) description = me.StringField() price = me.DecimalField(default=0.0)
class BaseDocument(mongoengine.Document): created_on = mongoengine.DateTimeField() modified_on = mongoengine.DateTimeField() status_modified_on = mongoengine.DateTimeField() created_by = mongoengine.EmbeddedDocumentField(User) modified_by = mongoengine.EmbeddedDocumentField(User) status_modified_by = mongoengine.EmbeddedDocumentField(User) is_active = mongoengine.BooleanField(default=False) inactive_reason = mongoengine.StringField() tags = mongoengine.ListField(mongoengine.StringField(choices=CHOICES["tags"])) meta = { 'allow_inheritance': True, 'abstract': True, 'indexes': ['created_on', 'modified_on', 'is_active', 'tags'], 'queryset_class': CustomQuerySet } def __str__(self): return str(self.uid) def save(self): current_datetime = datetime.now() if hasattr(self, 'created_on') and not self.created_on: self.created_on = current_datetime self.created_by = User(username=g.user_info['username'], user_id=g.user_info['user_id']) if hasattr(g, 'user_info') else User(username="******", user_id="0") self.modified_on = current_datetime self.modified_by = User(username=g.user_info['username'], user_id=g.user_info['user_id']) if hasattr(g, 'user_info') else User(username="******", user_id="0") if hasattr(self, '_changed_fields') and 'is_active' in self._changed_fields: self.status_modified_on = current_datetime self.status_modified_by = User(username=g.user_info['username'], user_id=g.user_info['user_id']) if hasattr(g, 'user_info') else User(username="******", user_id="0") return super().save() def update(self, **kwargs): current_datetime = datetime.now() kwargs['upsert'] = False # upsert is set to false to make created_on and modified_on flag work properly kwargs['modified_on'] = current_datetime kwargs['modified_by'] = User(username=g.user_info['username'], user_id=g.user_info['user_id']) if hasattr(g, 'user_info') else User(username="******", user_id="0") if 'is_active' in kwargs: kwargs['status_modified_on'] = current_datetime kwargs['status_modified_by'] = kwargs['modified_by'] return super().update(**kwargs) def update_and_signal(self, **values): signals.me_pre_update.send(self.__class__, **{'document': self, 'update': values}) obj = self.update(**values) signals.me_post_update.send(self.__class__, **{'document': self, 'update': values}) return obj @classmethod def create_or_update(cls, query, values): try: instance = cls.objects.get(**query) except mongoengine.DoesNotExist: instance = cls(**values) instance.save() else: instance.update(**values) instance.save() return instance @classmethod def create_or_update_and_signal(cls, query, values): try: instance = cls.objects.get(**query) except mongoengine.DoesNotExist: instance = cls(**values) signals.me_pre_update.send(instance.__class__, **{'document': instance}) instance.save() else: signals.me_pre_update.send(instance.__class__, **{'document': instance, 'update': values}) instance.update(**values) instance.save() signals.me_post_update.send(instance.__class__, **{'document': instance}) return instance
class MUserFeedNotification(mongo.Document): '''A user's notifications of a single feed.''' user_id = mongo.IntField() feed_id = mongo.IntField() frequency = mongoengine_fields.IntEnumField(NotificationFrequency) is_focus = mongo.BooleanField() last_notification_date = mongo.DateTimeField(default=datetime.datetime.now) is_email = mongo.BooleanField() is_web = mongo.BooleanField() is_ios = mongo.BooleanField() is_android = mongo.BooleanField() ios_tokens = mongo.ListField(mongo.StringField(max_length=1024)) meta = { 'collection': 'notifications', 'indexes': [ 'feed_id', { 'fields': ['user_id', 'feed_id'], 'unique': True, 'types': False, } ], 'allow_inheritance': False, } def __unicode__(self): notification_types = [] if self.is_email: notification_types.append('email') if self.is_web: notification_types.append('web') if self.is_ios: notification_types.append('ios') if self.is_android: notification_types.append('android') return "%s/%s: %s -> %s" % ( User.objects.get(pk=self.user_id).username, Feed.get_by_id(self.feed_id), ','.join(notification_types), self.last_notification_date, ) @classmethod def feed_has_users(cls, feed_id): return cls.users_for_feed(feed_id).count() @classmethod def users_for_feed(cls, feed_id): notifications = cls.objects.filter(feed_id=feed_id) return notifications @classmethod def feeds_for_user(cls, user_id): notifications = cls.objects.filter(user_id=user_id) notifications_by_feed = {} for feed in notifications: notifications_by_feed[feed.feed_id] = { 'notification_types': [], 'notification_filter': "focus" if feed.is_focus else "unread", } if feed.is_email: notifications_by_feed[ feed.feed_id]['notification_types'].append('email') if feed.is_web: notifications_by_feed[ feed.feed_id]['notification_types'].append('web') if feed.is_ios: notifications_by_feed[ feed.feed_id]['notification_types'].append('ios') if feed.is_android: notifications_by_feed[ feed.feed_id]['notification_types'].append('android') return notifications_by_feed @classmethod def push_feed_notifications(cls, feed_id, new_stories, force=False): feed = Feed.get_by_id(feed_id) notifications = MUserFeedNotification.users_for_feed(feed.pk) logging.debug( " ---> [%-30s] ~FCPushing out notifications to ~SB%s users~SN for ~FB~SB%s stories" % (feed, len(notifications), new_stories)) r = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL) latest_story_hashes = r.zrange("zF:%s" % feed.pk, -1 * new_stories, -1) mstories = MStory.objects.filter( story_hash__in=latest_story_hashes).order_by('-story_date') stories = Feed.format_stories(mstories) total_sent_count = 0 for user_feed_notification in notifications: sent_count = 0 last_notification_date = user_feed_notification.last_notification_date try: usersub = UserSubscription.objects.get( user=user_feed_notification.user_id, feed=user_feed_notification.feed_id) except UserSubscription.DoesNotExist: continue classifiers = user_feed_notification.classifiers(usersub) if classifiers == None: logging.debug("Has no usersubs") continue for story in stories: if sent_count >= 3: logging.debug("Sent too many, ignoring...") continue if story['story_date'] <= last_notification_date and not force: logging.debug( "Story date older than last notification date: %s <= %s" % (story['story_date'], last_notification_date)) continue if story[ 'story_date'] > user_feed_notification.last_notification_date: user_feed_notification.last_notification_date = story[ 'story_date'] user_feed_notification.save() story['story_content'] = HTMLParser().unescape( story['story_content']) sent = user_feed_notification.push_story_notification( story, classifiers, usersub) if sent: sent_count += 1 total_sent_count += 1 return total_sent_count, len(notifications) def classifiers(self, usersub): classifiers = {} if usersub.is_trained: classifiers['feeds'] = list( MClassifierFeed.objects(user_id=self.user_id, feed_id=self.feed_id, social_user_id=0)) classifiers['authors'] = list( MClassifierAuthor.objects(user_id=self.user_id, feed_id=self.feed_id)) classifiers['titles'] = list( MClassifierTitle.objects(user_id=self.user_id, feed_id=self.feed_id)) classifiers['tags'] = list( MClassifierTag.objects(user_id=self.user_id, feed_id=self.feed_id)) return classifiers def title_and_body(self, story, usersub): def replace_with_newlines(element): text = '' for elem in element.recursiveChildGenerator(): if isinstance(elem, types.StringTypes): text += elem elif elem.name == 'br': text += '\n' elif elem.name == 'p': text += '\n\n' return text feed_title = usersub.user_title or usersub.feed.feed_title # title = "%s: %s" % (feed_title, story['story_title']) title = feed_title subtitle = story['story_title'] # body = HTMLParser().unescape(strip_tags(story['story_content'])) soup = BeautifulSoup(story['story_content'].strip()) # print story['story_content'] body = replace_with_newlines(soup) body = truncate_chars(body.strip(), 1600) return title, subtitle, body def push_story_notification(self, story, classifiers, usersub): story_score = self.story_score(story, classifiers) if self.is_focus and story_score <= 0: logging.debug("Is focus, but story is hidden") return False elif story_score < 0: logging.debug("Is unread, but story is hidden") return False user = User.objects.get(pk=self.user_id) logging.user( user, "~FCSending push notification: %s/%s (score: %s)" % (story['story_title'][:40], story['story_hash'], story_score)) self.send_web(story, user) self.send_ios(story, user, usersub) self.send_android(story) self.send_email(story, usersub) return True def send_web(self, story, user): if not self.is_web: return r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL) r.publish( user.username, 'notification:%s,%s' % (story['story_hash'], story['story_title'])) def send_ios(self, story, user, usersub): if not self.is_ios: return apns = APNs( use_sandbox=True, cert_file='/srv/newsblur/config/certificates/aps_development.pem', key_file='/srv/newsblur/config/certificates/aps_development.pem') tokens = MUserNotificationTokens.get_tokens_for_user(self.user_id) title, subtitle, body = self.title_and_body(story, usersub) image_url = None if len(story['image_urls']): image_url = story['image_urls'][0] # print image_url for token in tokens.ios_tokens: logging.user( user, '~BMStory notification by iOS: ~FY~SB%s~SN~BM~FY/~SB%s' % (story['story_title'][:50], usersub.feed.feed_title[:50])) payload = Payload(alert={ 'title': title, 'subtitle': subtitle, 'body': body }, category="STORY_CATEGORY", mutable_content=True, custom={ 'story_hash': story['story_hash'], 'story_feed_id': story['story_feed_id'], 'image_url': image_url, }) apns.gateway_server.send_notification(token, payload) def send_android(self, story): if not self.is_android: return def send_email(self, story, usersub): if not self.is_email: return feed = usersub.feed story_content = self.sanitize_story(story['story_content']) params = { "story": story, "story_content": story_content, "feed": feed, "feed_title": usersub.user_title or feed.feed_title, "favicon_border": feed.favicon_color, } from_address = '*****@*****.**' to_address = '%s <%s>' % (usersub.user.username, usersub.user.email) text = render_to_string('mail/email_story_notification.txt', params) html = render_to_string('mail/email_story_notification.xhtml', params) subject = '%s: %s' % (usersub.user_title or usersub.feed.feed_title, story['story_title']) subject = subject.replace('\n', ' ') msg = EmailMultiAlternatives(subject, text, from_email='NewsBlur <%s>' % from_address, to=[to_address]) msg.attach_alternative(html, "text/html") try: msg.send() except boto.ses.connection.ResponseError, e: logging.user(usersub.user, '~BMStory notification by email error: ~FR%s' % e) logging.user( usersub.user, '~BMStory notification by email: ~FY~SB%s~SN~BM~FY/~SB%s' % (story['story_title'][:50], usersub.feed.feed_title[:50]))
class MStatistics(mongo.Document): key = mongo.StringField(unique=True) value = mongo.StringField() meta = { 'collection': 'statistics', 'allow_inheritance': False, 'indexes': ['key'], } def __unicode__(self): return "%s: %s" % (self.key, self.value) @classmethod def all(cls): values = dict([(stat.key, stat.value) for stat in cls.objects.all()]) for key, value in values.items(): if key in ('avg_time_taken', 'sites_loaded'): values[key] = json.decode(value) elif key in ('feeds_fetched', 'premium_users', 'standard_users', 'latest_sites_loaded', 'max_sites_loaded'): values[key] = int(value) elif key in ('latest_avg_time_taken', 'max_avg_time_taken'): values[key] = float(value) return values @classmethod def collect_statistics(cls): last_day = datetime.datetime.now() - datetime.timedelta(hours=24) cls.collect_statistics_feeds_fetched(last_day) cls.collect_statistics_premium_users(last_day) cls.collect_statistics_standard_users(last_day) cls.collect_statistics_sites_loaded(last_day) @classmethod def collect_statistics_feeds_fetched(cls, last_day=None): if not last_day: last_day = datetime.datetime.now() - datetime.timedelta(hours=24) feeds_fetched = MFeedFetchHistory.objects( fetch_date__gte=last_day).count() cls.objects(key='feeds_fetched').update_one(upsert=True, key='feeds_fetched', value=feeds_fetched) return feeds_fetched @classmethod def collect_statistics_premium_users(cls, last_day=None): if not last_day: last_day = datetime.datetime.now() - datetime.timedelta(hours=24) premium_users = Profile.objects.filter(last_seen_on__gte=last_day, is_premium=True).count() cls.objects(key='premium_users').update_one(upsert=True, key='premium_users', value=premium_users) return premium_users @classmethod def collect_statistics_standard_users(cls, last_day=None): if not last_day: last_day = datetime.datetime.now() - datetime.timedelta(hours=24) standard_users = Profile.objects.filter(last_seen_on__gte=last_day, is_premium=False).count() cls.objects(key='standard_users').update_one(upsert=True, key='standard_users', value=standard_users) return standard_users @classmethod def collect_statistics_sites_loaded(cls, last_day=None): if not last_day: last_day = datetime.datetime.now() - datetime.timedelta(hours=24) now = datetime.datetime.now() sites_loaded = [] avg_time_taken = [] for hour in range(24): start_hours_ago = now - datetime.timedelta(hours=hour) end_hours_ago = now - datetime.timedelta(hours=hour + 1) aggregates = dict(count=Count('loadtime'), avg=Avg('loadtime')) load_times = FeedLoadtime.objects.filter( date_accessed__lte=start_hours_ago, date_accessed__gte=end_hours_ago).aggregate(**aggregates) sites_loaded.append(load_times['count'] or 0) avg_time_taken.append(load_times['avg'] or 0) sites_loaded.reverse() avg_time_taken.reverse() values = ( ('sites_loaded', json.encode(sites_loaded)), ('avg_time_taken', json.encode(avg_time_taken)), ('latest_sites_loaded', sites_loaded[-1]), ('latest_avg_time_taken', avg_time_taken[-1]), ('max_sites_loaded', max(sites_loaded)), ('max_avg_time_taken', max(1, max(avg_time_taken))), ) for key, value in values: cls.objects(key=key).update_one(upsert=True, key=key, value=value)
class FakeModelDB(stormbase.StormBaseDB): state = me.StringField(required=True)
class ExclusionModel(gj.Document): to_json_exclude = db.StringField(exclude_to_json=True) from_json_exclude = db.IntField(exclude_from_json=True) json_exclude = db.StringField(exclude_json=True) required = db.StringField(required=True)
class Messages(me.Document): meta = {'collection': 'questions'} timestamp = me.DecimalField(precision=6) user = me.StringField() data = me.DictField()