class Questions(me.Document):
meta = {'collection': 'questions'}
user = me.StringField(required=True)
text = me.StringField(required=True)
date = me.DateTimeField(required=True)
class User(mongoengine.EmbeddedDocument):
username = mongoengine.StringField()
user_id = mongoengine.StringField()
class User(me.Document):
# The fields needed to display a user's name and profile picture.
CORE_FIELDS = ['first_name', 'last_name', 'fbid', 'email']
class JoinSource(object):
FACEBOOK = 1
EMAIL = 2
class UserCreationError(Exception):
pass
meta = {
'indexes': [
'fb_access_token',
'fbid',
# TODO(mack): need to create the 'api_key' index on prod
'api_key',
# Allow users with email=None, but non-None emails must be unique
{
'fields': ['email'],
'unique': True,
'sparse': True,
},
'referrer_id',
],
}
# Randomly generated ID used to access some subset of user's information
# without going through any ACL. Used for e.g. sharing schedules with non
# flow users.
#
# e.g. A8RLLZTMX
secret_id = me.StringField()
# TODO(mack): join_date should be encapsulate in _id, but store it
# for now, just in case; can remove it when sure that info is in _id
join_date = me.DateTimeField(required=True)
join_source = me.IntField(required=True,
choices=[JoinSource.FACEBOOK, JoinSource.EMAIL])
referrer_id = me.ObjectIdField(required=False)
# eg. Mack
first_name = me.StringField(required=True)
middle_name = me.StringField()
# eg. Duan
last_name = me.StringField(required=True)
# TODO(mack): check if facebook always returns gender field
gender = me.StringField(choices=['male', 'female'])
# eg. 1647810326
fbid = me.StringField()
# http://stackoverflow.com/questions/4408945/what-is-the-length-of-the-access-token-in-facebook-oauth2
fb_access_token = me.StringField(max_length=255)
fb_access_token_expiry_date = me.DateTimeField()
# The token expired due to de-auth, logging out, etc (ie. not time expired)
fb_access_token_invalid = me.BooleanField(default=False)
email = me.EmailField()
password = me.StringField()
# eg. list of user objectids, could be friends from sources besides
# facebook
friend_ids = me.ListField(me.ObjectIdField())
# eg. list of fbids of friends from facebook, not necessarily all of whom
# use the site
friend_fbids = me.ListField(me.StringField())
birth_date = me.DateTimeField()
last_visited = me.DateTimeField()
# TODO(mack): consider using SequenceField()
num_visits = me.IntField(min_value=0, default=0)
# The last time the user visited the onboarding page
last_show_onboarding = me.DateTimeField()
# The last time the user was shown the import schedule view
last_show_import_schedule = me.DateTimeField()
# eg. mduan or 20345619 ?
student_id = me.StringField()
# eg. university_of_waterloo ?
school_id = me.StringField()
# eg. software_engineering ?
# TODO(mack): should store program_id, not program_name
# program_id = me.StringField()
program_name = me.StringField()
# List of UserCourse.id's
course_history = me.ListField(me.ObjectIdField())
# TODO(mack): figure out why last_term_id was commented out in
# a prior diff: #260f174
# Deprecated
last_term_id = me.StringField()
# Deprecated
last_program_year_id = me.StringField()
# Track the number of times the user has invited friends
# (So we can award points if they have)
num_invites = me.IntField(min_value=0, default=0)
# The number of points this user has. Point are awarded for a number of
# actions such as reviewing courses, inviting friends. This is a cached
# point total. It will be calculated once a day with aggregator.py
num_points = me.IntField(min_value=0, default=0)
is_admin = me.BooleanField(default=False)
# TODO(mack): refactor this into something maintainable
sent_exam_schedule_notifier_email = me.BooleanField(default=False)
sent_velocity_demo_notifier_email = me.BooleanField(default=False)
sent_raffle_notifier_email = me.BooleanField(default=False)
sent_raffle_end_notifier_email = me.BooleanField(default=False)
sent_schedule_sharing_notifier_email = me.BooleanField(default=False)
sent_course_enrollment_feb_8_email = me.BooleanField(default=False)
sent_referral_contest_email = me.BooleanField(default=False)
sent_referral_contest_end_email = me.BooleanField(default=False)
sent_welcome_email = me.BooleanField(default=False)
email_unsubscribed = me.BooleanField(default=False)
# Note: Backfilled on night of Nov. 29th, 2012
transcripts_imported = me.IntField(min_value=0, default=0)
schedules_imported = me.IntField(min_value=0, default=0)
last_bad_schedule_paste = me.StringField()
last_good_schedule_paste = me.StringField()
last_bad_schedule_paste_date = me.DateTimeField()
last_good_schedule_paste_date = me.DateTimeField()
# Whether this user imported a schedule when it was still broken and we
# should email them to apologize
schedule_sorry = me.BooleanField(default=False)
# API key that grants user to login_required APIs
api_key = me.StringField()
last_prompted_for_review = me.DateTimeField(default=datetime.datetime.min)
voted_course_review_ids = me.ListField(me.StringField())
voted_prof_review_ids = me.ListField(me.StringField())
# Scholarships where a user has clicked: "Remove from profile"
closed_scholarship_ids = me.ListField(me.StringField())
@property
def name(self):
return '%s %s' % (self.first_name, self.last_name)
def save(self, *args, **kwargs):
# TODO(mack): If _changed_fields attribute does not exist, it mean
# document has been saved yet. Just need to verify. In this case,
# we could just check if id has been set
first_save = not hasattr(self, '_changed_fields')
if first_save:
# TODO(Sandy): We're assuming people won't unfriend anyone.
# Fix this later?
# TODO(mack): this isn't safe against race condition of both
# friends signing up at same time
#print 'friend_fbids', self.friend_fbids
friends = (User.objects(fbid__in=self.friend_fbids).only(
'id', 'friend_ids'))
self.friend_ids = [f.id for f in friends]
super(User, self).save(*args, **kwargs)
if first_save:
# TODO(mack): should do this asynchronously
# Using update rather than save because it should be more efficient
friends.update(add_to_set__friend_ids=self.id)
# TODO(mack): think of better way to cache value
@property
def course_ids(self):
if not hasattr(self, '_course_ids'):
user_courses = _user_course.UserCourse.objects(
id__in=self.course_history).only('course_id')
self._course_ids = [uc.course_id for uc in user_courses]
return self._course_ids
@property
def profile_pic_urls(self):
if self.fbid is not None:
urls = self._get_fb_pic_urls()
else:
urls = self._get_gravatar_pic_urls()
return urls
def _get_fb_pic_urls(self):
base_pic = "https://graph.facebook.com/%s/picture" % (self.fbid)
return {
'default': base_pic,
'large': '%s?type=large' % (base_pic),
'square': '%s?type=square' % (base_pic),
}
def _get_gravatar_pic_urls(self):
# Gravatar API: https://en.gravatar.com/site/implement/images/
# TODO(sandy): Serve our own default image instead of the mystery man
email_hash = hashlib.md5(self.email.strip().lower()).hexdigest()
base_pic = "https://secure.gravatar.com/avatar/%s?d=mm" % (email_hash)
return {
'default': "%s&size=%s" % (base_pic, "50"),
'large': "%s&size=%s" % (base_pic, "190"),
'square': "%s&size=%s" % (base_pic, "50"),
}
@property
def profile_url(self):
return '/profile/%s' % self.id
@property
def absolute_profile_url(self):
return '%s%s?admin=1' % (constants.RMC_HOST, self.profile_url)
@property
def short_program_name(self):
if self.program_name:
return self.program_name.split(',')[0]
return ''
@property
def has_course_history(self):
# TODO(Sandy): Using this to backfill transcripts imported,
# remove later
if len(self.course_history) == 0:
return False
for uc in self.get_user_courses():
if not _term.Term.is_shortlist_term(uc.term_id):
return True
return False
@property
def has_shortlisted(self):
for uc in self.get_user_courses():
if _term.Term.is_shortlist_term(uc.term_id):
return True
return False
@property
def has_schedule(self):
# TODO(Sandy): Actually this only works assuming users never remove
# their schedule and we'll have to do actual queries when 2013_05 comes
return self.schedules_imported > 0
@property
def should_renew_fb_token(self):
# Should renew FB token if it expired or will expire "soon".
future_date = datetime.datetime.now() + datetime.timedelta(
days=facebook.FB_FORCE_TOKEN_EXPIRATION_DAYS)
return (self.fb_access_token_expiry_date is None
or self.fb_access_token_expiry_date < future_date
or self.fb_access_token_invalid)
@property
def is_fb_token_expired(self):
return (self.fb_access_token_expiry_date is None
or self.fb_access_token_expiry_date < datetime.datetime.now()
or self.fb_access_token_invalid)
@property
def is_demo_account(self):
return self.fbid == constants.DEMO_ACCOUNT_FBID
@property
def last_schedule_paste(self):
return self.last_good_schedule_paste or self.last_bad_schedule_paste
def get_user_courses(self):
return _user_course.UserCourse.objects(id__in=self.course_history)
@classmethod
def cls_mutual_courses_redis_key(cls, user_id_one, user_id_two):
if user_id_one < user_id_two:
first_id = user_id_one
second_id = user_id_two
else:
first_id = user_id_two
second_id = user_id_one
return 'mutual_courses:%s:%s' % (first_id, second_id)
def mutual_courses_redis_key(self, other_user_id):
return User.cls_mutual_courses_redis_key(self.id, other_user_id)
def get_mutual_course_ids(self, redis):
# fetch mutual friends from redis
pipe = redis.pipeline()
# Show mutual courses between the viewing user and the friends of the
# profile user
for friend_id in self.friend_ids:
pipe.smembers(self.mutual_courses_redis_key(friend_id))
mutual_course_ids_per_user = pipe.execute()
zipped = itertools.izip(self.friend_ids, mutual_course_ids_per_user)
mutual_course_ids_by_friend = {}
for friend_id, mutual_course_ids in zipped:
mutual_course_ids_by_friend[friend_id] = mutual_course_ids
return mutual_course_ids_by_friend
def cache_mutual_course_ids(self, redis):
friends = User.objects(id__in=self.friend_ids).only('course_history')
friend_map = {}
for friend in friends:
friend_map[friend.id] = friend
my_course_ids = set(self.course_ids)
for friend in friends:
mutual_course_ids = my_course_ids.intersection(friend.course_ids)
if mutual_course_ids:
redis_key = self.mutual_courses_redis_key(friend.id)
redis.sadd(redis_key, *list(mutual_course_ids))
def remove_mutual_course_ids(self, redis):
pipe = redis.pipeline()
for friend_id in self.friend_ids:
pipe.delete(self.mutual_courses_redis_key(friend_id))
return pipe.execute()
def get_latest_program_year_id(self):
latest_term_uc = None
for uc_dict in self.get_user_courses():
# Ignore untaken courses or shortlisted courses
if uc_dict['term_id'] > util.get_current_term_id():
continue
if not latest_term_uc:
latest_term_uc = uc_dict
elif uc_dict['term_id'] > latest_term_uc['term_id']:
latest_term_uc = uc_dict
if latest_term_uc:
return latest_term_uc['program_year_id']
return None
def get_friends(self):
"""Gets basic info for each of this user's friends."""
return User.objects(id__in=self.friend_ids).only(
*(User.CORE_FIELDS +
['id', 'num_points', 'num_invites', 'program_name']))
def rated_review(self, review_id, review_type):
if review_type == 'course':
return review_id in self.voted_course_review_ids
else:
return review_id in self.voted_prof_review_ids
def to_dict(self, extended=True, include_course_ids=False):
user_dict = {
'id': self.id,
'fbid': self.fbid,
'first_name': self.first_name,
'last_name': self.last_name,
'name': self.name,
'profile_pic_urls': self.profile_pic_urls,
'program_name': self.short_program_name,
'num_invites': self.num_invites,
'num_points': self.num_points,
}
if extended:
user_dict.update({
'friend_ids': self.friend_ids,
'course_history': self.course_history,
})
if include_course_ids:
user_dict['course_ids'] = self.course_ids
return user_dict
# TODO(mack): make race condition safe?
def delete(self, *args, **kwargs):
# Remove this user from the friend lists of all friends
friends = User.objects(id__in=self.friend_ids)
friends.update(pull__friend_ids=self.id)
# Delete all their user course objects
_user_course.UserCourse.objects(user_id=self.id).delete()
# Delete all their UserScheduleItem objects
_user_schedule_item.UserScheduleItem.objects(user_id=self.id).delete()
# TODO(mack): delete mutual course information from redis?
# should be fine for now since we are removing this user from their
# friends' friend_ids, and redis cache will be regenerated daily
# from aggregator.py
return super(User, self).delete(*args, **kwargs)
def to_review_author_dict(self, current_user, reveal_identity):
is_current_user = current_user and current_user.id == self.id
if reveal_identity:
return {
'id': self.id,
'name': 'You' if is_current_user else self.name,
'profile_pic_url': self.profile_pic_urls['square'],
}
else:
return {'program_name': self.short_program_name}
def invite_friend(self, redis):
self.num_invites += 1
if self.num_invites == 1:
self.award_points(_points.PointSource.FIRST_INVITE, redis)
def award_points(self, points, redis):
self.num_points += points
redis.incr('total_points', points)
def update_fb_friends(self, fbids):
self.friend_fbids = fbids
fb_friends = (User.objects(fbid__in=self.friend_fbids).only(
'id', 'friend_ids'))
# We have friends from only Facebook right now, so just set it
self.friend_ids = [f.id for f in fb_friends]
def get_schedule_item_dicts(self, exam_objs=None):
"""Gets all schedule items for this user starting no later than a year
ago.
Args:
exam_objs: Optional exam objects to convert to UserScheduleItem and
add to return list.
Returns: a list of UserScheduleItem models as dicts.
"""
one_year_ago = datetime.datetime.now() - datetime.timedelta(days=365)
schedule_item_objs = _user_schedule_item.UserScheduleItem.objects(
user_id=self.id, start_date__gte=one_year_ago)
dicts = [si.to_dict() for si in schedule_item_objs]
if exam_objs:
dicts.extend(e.to_schedule_obj().to_dict() for e in exam_objs)
return dicts
def get_failed_schedule_item_dicts(self):
one_year_ago = datetime.datetime.now() - datetime.timedelta(days=365)
schedule_item_objs = _user_schedule_item.FailedScheduleItem.objects(
user_id=self.id, parsed_date__gte=one_year_ago)
return [si.to_dict() for si in schedule_item_objs]
def get_all_schedule_items(self):
return _user_schedule_item.UserScheduleItem.objects(user_id=self.id)
def get_current_term_exams(self, current_term_course_ids=None):
if not current_term_course_ids:
ucs = (self.get_user_courses().filter(
term_id=util.get_current_term_id()).only('course_id'))
current_term_course_ids = [uc.course_id for uc in ucs]
return _exam.Exam.objects(course_id__in=current_term_course_ids)
def get_secret_id(self):
# TODO(jlfwong): This is possibly a race condition...
if self.secret_id is None:
self.secret_id = util.generate_secret_id()
self.save()
return self.secret_id
def add_course(self, course_id, term_id, program_year_id=None):
"""Creates a UserCourse and adds it to the user's course_history.
Idempotent.
Returns the resulting UserCourse.
"""
user_course = _user_course.UserCourse.objects(
user_id=self.id, course_id=course_id).first()
if user_course is None:
if _course.Course.objects.with_id(course_id) is None:
# Non-existant course according to our data
rmclogger.log_event(rmclogger.LOG_CATEGORY_DATA_MODEL,
rmclogger.LOG_EVENT_UNKNOWN_COURSE_ID,
course_id)
return None
user_course = _user_course.UserCourse(
user_id=self.id,
course_id=course_id,
term_id=term_id,
program_year_id=program_year_id,
)
else:
# Record only the latest attempt for duplicate/failed courses
if (term_id > user_course.term_id
or user_course.term_id == _term.Term.SHORTLIST_TERM_ID):
user_course.term_id = term_id
user_course.program_year_id = program_year_id
user_course.save()
if user_course.id not in self.course_history:
self.course_history.append(user_course.id)
self.save()
return user_course
# Generate a random api key granting this user to access '/api/' routes
def grant_api_key(self):
uuid_ = uuid.uuid4()
md5 = hashlib.md5()
md5.update(str(uuid_))
microsecs = int(time.time() * 1000000)
raw_api_key = str(microsecs) + md5.hexdigest()
self.api_key = base64.b64encode(raw_api_key)
self.save()
return self.api_key
def next_course_to_review(self):
user_courses = _user_course.UserCourse.objects(user_id=self.id)
return _user_course.UserCourse.select_course_to_review(user_courses)
def should_prompt_review(self):
now = datetime.datetime.now()
elapsed = min(now - self.last_prompted_for_review,
now - self.join_date)
return elapsed.days > PROMPT_TO_REVIEW_DELAY_DAYS
@staticmethod
def auth_user(email, password):
"""Returns the authenticated user or None."""
user = User.objects(email=email)
if not user:
return None
# TODO(sandy): Since we added a unique index on email, this shouldn't
# happen anymore. But keep this around for a bit, in case something
# messes up [Apr 8, 2014]
if user.count() > 1:
logging.error('Multiple email addressed matched: %s' % email)
return None
user = user.first()
# TODO(sandy): Provide more helpful errors for users signed up with fb
if (not user.password
or not bcrypt.check_password_hash(user.password, password)):
return None
return user
@staticmethod
def create_new_user_from_email(first_name, last_name, email, password):
if len(password) < PASSWORD_MIN_LENGTH:
raise User.UserCreationError(
'Passwords must be at least 8 characters long.')
user = User(
email=email,
first_name=first_name,
join_date=datetime.datetime.now(),
join_source=User.JoinSource.EMAIL,
last_name=last_name,
password=bcrypt.generate_password_hash(password,
rounds=BCRYPT_ROUNDS),
)
try:
user.save()
except me.base.ValidationError as e:
if 'email' in e.errors:
raise User.UserCreationError('Oops, that email is invalid.')
raise
except me.queryset.NotUniqueError as e:
raise User.UserCreationError(
'That email is already signed up.'
' (Maybe you already signed up with Facebook?)')
return user
def __repr__(self):
return "<User: %s>" % self.name.encode('utf-8')
class Address(BaseModel):
state = mongoengine.StringField()
county = mongoengine.StringField()
zipcode = mongoengine.StringField()
street = mongoengine.StringField()
address = mongoengine.StringField()
class PollingSchedule(me.Document):
meta = {
'allow_inheritance': True,
'strict': False,
}
# We use a unique name for easy identification and to avoid running the
# same schedule twice. The name is autopopulated during the invocation of
# the `clean` method.
name = me.StringField(unique=True)
# The following fields are defined in celerybeatmongo.models.PeriodicTask.
# Here, we define no fields in the base class, and expect subclasses to
# either define their fields, or simply use properties.
# task = me.StringField(required=True)
# args = me.ListField()
# kwargs = me.DictField()
# Scheduling information. Don't edit them directly, just use the model
# methods.
default_interval = me.EmbeddedDocumentField(
PollingInterval, required=True, default=PollingInterval(every=0))
override_intervals = me.EmbeddedDocumentListField(PollingInterval)
# Optional arguments.
queue = me.StringField()
exchange = me.StringField()
routing_key = me.StringField()
soft_time_limit = me.IntField()
# Used internally by the scheduler.
last_run_at = me.DateTimeField()
total_run_count = me.IntField(min_value=0)
run_immediately = me.BooleanField()
def get_name(self):
"""Construct name based on self.task"""
try:
return self.task.split('.')[-1]
except NotImplementedError:
return '%s: No task specified.' % self.__class__.__name__
def clean(self):
"""Automatically set value of name"""
self.name = self.get_name()
@property
def task(self):
"""Return task name for this schedule
Subclasses should define an attribute, property or field to do this.
"""
raise NotImplementedError()
@property
def args(self):
"""Return task args for this schedule"""
return [str(self.id)]
@property
def kwargs(self):
"""Return task kwargs for this schedule"""
return {}
@property
def enabled(self):
"""Whether this task is currently enabled or not"""
return bool(self.interval.timedelta)
@property
def interval(self):
"""Merge multiple intervals into one
Returns a dynamic PollingInterval, with the highest frequency of any
override schedule or the default schedule.
"""
interval = self.default_interval
for i in self.override_intervals:
if not i.expired():
if not interval.timedelta or i.timedelta < interval.timedelta:
interval = i
return interval
@property
def schedule(self):
"""Return a celery schedule instance
This is used internally by celerybeatmongo scheduler
"""
return celery.schedules.schedule(self.interval.timedelta)
@property
def expires(self):
return None
def add_interval(self, interval, ttl=300, name=''):
"""Add an override schedule to the scheduled task
Override schedules must define an interval in seconds, as well as a
TTL (time to live), also in seconds. Override schedules cannot be
removed, so short TTL's should be used. You can however add a new
override schedule again, thus practically extending the time where an
override is in effect.
Override schedules can only increase, not decrease frequency of the
schedule, in relation to that define in the `default_interval`.
"""
assert isinstance(interval, int) and interval > 0
assert isinstance(ttl, int) and 0 < ttl < 3600
expires = datetime.datetime.now() + datetime.timedelta(seconds=ttl)
self.override_intervals.append(
PollingInterval(name=name, expires=expires, every=interval))
def cleanup_expired_intervals(self):
"""Remove override schedules that have expired"""
self.override_intervals = [
override for override in self.override_intervals
if not override.expired()
]
def set_default_interval(self, interval):
"""Set default interval
This is the interval used for this schedule, if there is no active
override schedule with a smaller interval. The default interval never
expires. To disable a task, simply set `enabled` equal to False.
"""
self.default_interval = PollingInterval(name='default', every=interval)
def __unicode__(self):
return "%s %s" % (self.get_name(), self.interval or '(no interval)')
class Video(mongoengine.Document):
name = mongoengine.StringField(required=True)
theme = mongoengine.StringField(required=True)
thumbs_up = mongoengine.IntField(default=0)
thumbs_down = mongoengine.IntField(default=0)
class County(BaseModel):
state = mongoengine.StringField()
county = mongoengine.StringField()
meta = {"collection": Model.county}
class MUserStory(mongo.Document):
"""
Stories read by the user. These are deleted as the mark_read_date for the
UserSubscription passes the UserStory date.
"""
user_id = mongo.IntField()
feed_id = mongo.IntField()
read_date = mongo.DateTimeField()
story_id = mongo.StringField(unique_with=('user_id', 'feed_id'))
story_date = mongo.DateTimeField()
story = mongo.ReferenceField(MStory)
found_story = mongo.GenericReferenceField()
meta = {
'collection':
'userstories',
'indexes': [
{
'fields': ('user_id', 'feed_id', 'story_id'),
'unique': True
},
('feed_id', 'story_id'), # Updating stories with new guids
('feed_id', 'story_date'), # Trimming feeds
],
'allow_inheritance':
False,
'index_drop_dups':
True,
}
def save(self, *args, **kwargs):
self.sync_redis()
super(MUserStory, self).save(*args, **kwargs)
def delete(self, *args, **kwargs):
self.remove_from_redis()
super(MUserStory, self).delete(*args, **kwargs)
@property
def guid_hash(self):
return hashlib.sha1(self.story_id).hexdigest()
@classmethod
def delete_old_stories(cls, feed_id):
UNREAD_CUTOFF = datetime.datetime.utcnow() - datetime.timedelta(
days=settings.DAYS_OF_UNREAD)
cls.objects(feed_id=feed_id, story_date__lte=UNREAD_CUTOFF).delete()
@classmethod
def delete_marked_as_read_stories(cls,
user_id,
feed_id,
mark_read_date=None):
if not mark_read_date:
usersub = UserSubscription.objects.get(user__pk=user_id,
feed__pk=feed_id)
mark_read_date = usersub.mark_read_date
# Next line forces only old read stories to be removed, just in case newer stories
# come in as unread because they're being shared.
mark_read_date = datetime.datetime.utcnow() - datetime.timedelta(
days=settings.DAYS_OF_UNREAD)
cls.objects(user_id=user_id,
feed_id=feed_id,
read_date__lte=mark_read_date).delete()
@property
def story_db_id(self):
if self.story:
return self.story.id
elif self.found_story:
if '_ref' in self.found_story:
return self.found_story['_ref'].id
elif hasattr(self.found_story, 'id'):
return self.found_story.id
story, found_original = MStory.find_story(self.feed_id, self.story_id)
if story:
if found_original:
self.story = story
else:
self.found_story = story
self.save()
return story.id
def sync_redis(self, r=None):
if not r:
r = redis.Redis(connection_pool=settings.REDIS_STORY_POOL)
if self.story_db_id:
all_read_stories_key = 'RS:%s' % (self.user_id)
r.sadd(all_read_stories_key, self.story_db_id)
read_story_key = 'RS:%s:%s' % (self.user_id, self.feed_id)
r.sadd(read_story_key, self.story_db_id)
r.expire(read_story_key, settings.DAYS_OF_UNREAD * 24 * 60 * 60)
def remove_from_redis(self):
r = redis.Redis(connection_pool=settings.REDIS_STORY_POOL)
if self.story_db_id:
r.srem('RS:%s' % self.user_id, self.story_db_id)
r.srem('RS:%s:%s' % (self.user_id, self.feed_id), self.story_db_id)
@classmethod
def sync_all_redis(cls):
read_stories = cls.objects.all()
for read_story in read_stories:
read_story.sync_redis()
class Snake:
registered_date = mongoengine.DateTimeField(default=datetime.now())
species = mongoengine.StringField(required=True)
class County(me.Document):
name = me.StringField(required=True)
geometry = me.MultiPolygonField(required=True)
class Jail(me.Document):
name = me.StringField(required=True)
location = me.PointField(required=True)
state = me.StringField()
class Category(me.Document):
title = me.StringField(min_length=1, max_length=512)
description = me.StringField(min_length=2, max_length=4096)
subcategories = me.ListField(me.ReferenceField('self'))
parent = me.ReferenceField('self', default=None)
class Artist(mongoengine.Document):
first_name = mongoengine.StringField(max_length=50)
last_name = mongoengine.StringField(max_length=50)
class User(me.Document):
meta = {'collection': 'questions'}
user = me.StringField(required=True)
admin = me.BooleanField(required=True)
class Material(mongoengine.EmbeddedDocument):
"""材料属性
"""
materialid = mongoengine.StringField(verbose_name="材料ID")
amount = mongoengine.IntField(verbose_name="数量", default=0)
class User(me.Document):
name = me.StringField()
email = me.StringField(nullable=False) #nao permite campo nulo
class User(me.Document):
"""
status : 'wait for approval' -> wait member approve this profile
: 'activate' -> this profile are approve
"""
meta = {'collection': 'users'}
username = me.StringField(required=True, unique=True)
password = me.StringField()
email = me.EmailField(required=True, unique=True)
first_name = me.StringField(max_length=100, required=True)
last_name = me.StringField(max_length=100, required=True)
display_name = me.StringField(max_length=250, required=True)
default_profile = me.StringField(default='pumbaa.coe.psu.ac.th')
online_profiles = me.ListField(me.EmbeddedDocumentField(Profile))
status = me.StringField(max_length=100,
required=True,
default='wait for approval')
registration_date = me.DateTimeField(required=True,
default=datetime.datetime.now)
updated_date = me.DateTimeField(required=True,
default=datetime.datetime.now)
approvers = me.ListField(me.EmbeddedDocumentField(Approver))
ip_address = me.StringField(max_length=100,
required=True,
default='0.0.0.0')
roles = me.ListField(me.ReferenceField('Role', dbref=True))
def get_display_name(self):
if self.display_name is not None:
return self.display_name
else:
return self.username
def get_tim_display_name(self):
tname = self.get_display_name().split(" ")
if len(tname) > 1:
return "%s. %s" % (tname[0][0], tname[1])
else:
return "%s" % (tname[0])
def set_password(self, password):
from pyramid.threadlocal import get_current_request
request = get_current_request()
self.password = request.secret_manager.get_hash_password(password)
def get_profile(self, domain):
for profile in self.online_profiles:
if profile.domain == domain:
return profile
def get_role(self, name):
for role in self.roles:
if role.name == name:
return role
def get_profile_picture(self, width=50):
if self.default_profile == 'pumbaa.coe.psu.ac.th':
return None
profile = self.get_profile(self.default_profile)
if profile.domain == 'facebook.com':
if '=' in profile.username:
username = profile.username.split('=')[-1]
else:
username = profile.username
return '<img src="https://graph.facebook.com/%s/picture" width="%d"/>' % (
username, width)
if profile.domain == 'twitter.com':
return '<img src="%s" width="%d"/>' % (
profile.profile_source['photos'][0]['value'], width)
if profile.domain == 'accounts.google.com':
return '<img src="%s" width="%d"/>' % (
profile.profile_source['photos'][0]['value'], width)
return None
def get_profile_url(self):
if self.default_profile == 'pumbaa.coe.psu.ac.th':
return '#'
profile = self.get_profile(self.default_profile)
if profile.domain == 'facebook.com':
return 'https://www.facebook.com/%s' % profile.username
if profile.domain == 'twitter.com':
return 'https://twitter.com/%s' % profile.username
if profile.domain == 'accounts.google.com':
return 'https://plus.google.com/%s' % profile.user_id
return '#'
class DiscreteMeasurement(Measurement):
result = mongoengine.StringField(required=True)
class State(BaseModel):
state = mongoengine.StringField()
meta = {"collection": Model.state}
class User(mongoengine.Document):
first_name = mongoengine.StringField(max_length=128, required=True)
last_name = mongoengine.StringField(max_length=128, required=True)
email = mongoengine.EmailField(max_length=128, required=True, unique=True)
class Zipcode(BaseModel):
state = mongoengine.StringField()
county = mongoengine.StringField()
zipcode = mongoengine.StringField()
meta = {"collection": Model.zipcode}
class Character(me.Document):
name = me.StringField(required=True)
height = me.StringField(required=True)
mass = me.StringField(required=True)
birth_year = me.StringField(required=True)
gender = me.StringField(required=True)
homeworld = me.StringField(required=True)
films = me.ListField(me.StringField(), required=True)
starships = me.ListField(me.StringField())
vehicles = me.ListField(me.StringField())
planets = me.ListField(me.StringField())
url = me.StringField(required=True)
@classmethod
def get_all(cls):
return cls.objects()
@classmethod
def get_by_id(cls, id):
return cls.objects(id=id)
@classmethod
def add(cls, name,height,mass,birth_year,gender, url, films, starships, vehicles, homeworld):
cls(
name=name,
height=height,
mass=mass,
birth_year=birth_year,
gender=gender,
homeworld=homeworld,
url=url,
films=[Films.get_by_url(film).to_json() for film in films],
starships=[Starships.get_by_url(starship).to_json() for starship in starships],
vehicles=[Vehicles.get_by_url(vehicle).to_json() for vehicle in vehicles],
planets=[Planets.get_by_url(homeworld).to_json()]
).save()
class DebugPollingSchedule(PollingSchedule):
task = 'mist.api.poller.tasks.debug'
value = me.StringField()
class Product(me.Document):
name = me.StringField(required=True)
description = me.StringField()
price = me.DecimalField(default=0.0)
class BaseDocument(mongoengine.Document):
created_on = mongoengine.DateTimeField()
modified_on = mongoengine.DateTimeField()
status_modified_on = mongoengine.DateTimeField()
created_by = mongoengine.EmbeddedDocumentField(User)
modified_by = mongoengine.EmbeddedDocumentField(User)
status_modified_by = mongoengine.EmbeddedDocumentField(User)
is_active = mongoengine.BooleanField(default=False)
inactive_reason = mongoengine.StringField()
tags = mongoengine.ListField(mongoengine.StringField(choices=CHOICES["tags"]))
meta = {
'allow_inheritance': True,
'abstract': True,
'indexes': ['created_on', 'modified_on', 'is_active', 'tags'],
'queryset_class': CustomQuerySet
}
def __str__(self):
return str(self.uid)
def save(self):
current_datetime = datetime.now()
if hasattr(self, 'created_on') and not self.created_on:
self.created_on = current_datetime
self.created_by = User(username=g.user_info['username'], user_id=g.user_info['user_id']) if hasattr(g, 'user_info') else User(username="******", user_id="0")
self.modified_on = current_datetime
self.modified_by = User(username=g.user_info['username'], user_id=g.user_info['user_id']) if hasattr(g, 'user_info') else User(username="******", user_id="0")
if hasattr(self, '_changed_fields') and 'is_active' in self._changed_fields:
self.status_modified_on = current_datetime
self.status_modified_by = User(username=g.user_info['username'], user_id=g.user_info['user_id']) if hasattr(g, 'user_info') else User(username="******", user_id="0")
return super().save()
def update(self, **kwargs):
current_datetime = datetime.now()
kwargs['upsert'] = False # upsert is set to false to make created_on and modified_on flag work properly
kwargs['modified_on'] = current_datetime
kwargs['modified_by'] = User(username=g.user_info['username'], user_id=g.user_info['user_id']) if hasattr(g, 'user_info') else User(username="******", user_id="0")
if 'is_active' in kwargs:
kwargs['status_modified_on'] = current_datetime
kwargs['status_modified_by'] = kwargs['modified_by']
return super().update(**kwargs)
def update_and_signal(self, **values):
signals.me_pre_update.send(self.__class__, **{'document': self, 'update': values})
obj = self.update(**values)
signals.me_post_update.send(self.__class__, **{'document': self, 'update': values})
return obj
@classmethod
def create_or_update(cls, query, values):
try:
instance = cls.objects.get(**query)
except mongoengine.DoesNotExist:
instance = cls(**values)
instance.save()
else:
instance.update(**values)
instance.save()
return instance
@classmethod
def create_or_update_and_signal(cls, query, values):
try:
instance = cls.objects.get(**query)
except mongoengine.DoesNotExist:
instance = cls(**values)
signals.me_pre_update.send(instance.__class__, **{'document': instance})
instance.save()
else:
signals.me_pre_update.send(instance.__class__, **{'document': instance, 'update': values})
instance.update(**values)
instance.save()
signals.me_post_update.send(instance.__class__, **{'document': instance})
return instance
class MUserFeedNotification(mongo.Document):
'''A user's notifications of a single feed.'''
user_id = mongo.IntField()
feed_id = mongo.IntField()
frequency = mongoengine_fields.IntEnumField(NotificationFrequency)
is_focus = mongo.BooleanField()
last_notification_date = mongo.DateTimeField(default=datetime.datetime.now)
is_email = mongo.BooleanField()
is_web = mongo.BooleanField()
is_ios = mongo.BooleanField()
is_android = mongo.BooleanField()
ios_tokens = mongo.ListField(mongo.StringField(max_length=1024))
meta = {
'collection':
'notifications',
'indexes': [
'feed_id', {
'fields': ['user_id', 'feed_id'],
'unique': True,
'types': False,
}
],
'allow_inheritance':
False,
}
def __unicode__(self):
notification_types = []
if self.is_email: notification_types.append('email')
if self.is_web: notification_types.append('web')
if self.is_ios: notification_types.append('ios')
if self.is_android: notification_types.append('android')
return "%s/%s: %s -> %s" % (
User.objects.get(pk=self.user_id).username,
Feed.get_by_id(self.feed_id),
','.join(notification_types),
self.last_notification_date,
)
@classmethod
def feed_has_users(cls, feed_id):
return cls.users_for_feed(feed_id).count()
@classmethod
def users_for_feed(cls, feed_id):
notifications = cls.objects.filter(feed_id=feed_id)
return notifications
@classmethod
def feeds_for_user(cls, user_id):
notifications = cls.objects.filter(user_id=user_id)
notifications_by_feed = {}
for feed in notifications:
notifications_by_feed[feed.feed_id] = {
'notification_types': [],
'notification_filter': "focus" if feed.is_focus else "unread",
}
if feed.is_email:
notifications_by_feed[
feed.feed_id]['notification_types'].append('email')
if feed.is_web:
notifications_by_feed[
feed.feed_id]['notification_types'].append('web')
if feed.is_ios:
notifications_by_feed[
feed.feed_id]['notification_types'].append('ios')
if feed.is_android:
notifications_by_feed[
feed.feed_id]['notification_types'].append('android')
return notifications_by_feed
@classmethod
def push_feed_notifications(cls, feed_id, new_stories, force=False):
feed = Feed.get_by_id(feed_id)
notifications = MUserFeedNotification.users_for_feed(feed.pk)
logging.debug(
" ---> [%-30s] ~FCPushing out notifications to ~SB%s users~SN for ~FB~SB%s stories"
% (feed, len(notifications), new_stories))
r = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL)
latest_story_hashes = r.zrange("zF:%s" % feed.pk, -1 * new_stories, -1)
mstories = MStory.objects.filter(
story_hash__in=latest_story_hashes).order_by('-story_date')
stories = Feed.format_stories(mstories)
total_sent_count = 0
for user_feed_notification in notifications:
sent_count = 0
last_notification_date = user_feed_notification.last_notification_date
try:
usersub = UserSubscription.objects.get(
user=user_feed_notification.user_id,
feed=user_feed_notification.feed_id)
except UserSubscription.DoesNotExist:
continue
classifiers = user_feed_notification.classifiers(usersub)
if classifiers == None:
logging.debug("Has no usersubs")
continue
for story in stories:
if sent_count >= 3:
logging.debug("Sent too many, ignoring...")
continue
if story['story_date'] <= last_notification_date and not force:
logging.debug(
"Story date older than last notification date: %s <= %s"
% (story['story_date'], last_notification_date))
continue
if story[
'story_date'] > user_feed_notification.last_notification_date:
user_feed_notification.last_notification_date = story[
'story_date']
user_feed_notification.save()
story['story_content'] = HTMLParser().unescape(
story['story_content'])
sent = user_feed_notification.push_story_notification(
story, classifiers, usersub)
if sent:
sent_count += 1
total_sent_count += 1
return total_sent_count, len(notifications)
def classifiers(self, usersub):
classifiers = {}
if usersub.is_trained:
classifiers['feeds'] = list(
MClassifierFeed.objects(user_id=self.user_id,
feed_id=self.feed_id,
social_user_id=0))
classifiers['authors'] = list(
MClassifierAuthor.objects(user_id=self.user_id,
feed_id=self.feed_id))
classifiers['titles'] = list(
MClassifierTitle.objects(user_id=self.user_id,
feed_id=self.feed_id))
classifiers['tags'] = list(
MClassifierTag.objects(user_id=self.user_id,
feed_id=self.feed_id))
return classifiers
def title_and_body(self, story, usersub):
def replace_with_newlines(element):
text = ''
for elem in element.recursiveChildGenerator():
if isinstance(elem, types.StringTypes):
text += elem
elif elem.name == 'br':
text += '\n'
elif elem.name == 'p':
text += '\n\n'
return text
feed_title = usersub.user_title or usersub.feed.feed_title
# title = "%s: %s" % (feed_title, story['story_title'])
title = feed_title
subtitle = story['story_title']
# body = HTMLParser().unescape(strip_tags(story['story_content']))
soup = BeautifulSoup(story['story_content'].strip())
# print story['story_content']
body = replace_with_newlines(soup)
body = truncate_chars(body.strip(), 1600)
return title, subtitle, body
def push_story_notification(self, story, classifiers, usersub):
story_score = self.story_score(story, classifiers)
if self.is_focus and story_score <= 0:
logging.debug("Is focus, but story is hidden")
return False
elif story_score < 0:
logging.debug("Is unread, but story is hidden")
return False
user = User.objects.get(pk=self.user_id)
logging.user(
user, "~FCSending push notification: %s/%s (score: %s)" %
(story['story_title'][:40], story['story_hash'], story_score))
self.send_web(story, user)
self.send_ios(story, user, usersub)
self.send_android(story)
self.send_email(story, usersub)
return True
def send_web(self, story, user):
if not self.is_web: return
r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL)
r.publish(
user.username,
'notification:%s,%s' % (story['story_hash'], story['story_title']))
def send_ios(self, story, user, usersub):
if not self.is_ios: return
apns = APNs(
use_sandbox=True,
cert_file='/srv/newsblur/config/certificates/aps_development.pem',
key_file='/srv/newsblur/config/certificates/aps_development.pem')
tokens = MUserNotificationTokens.get_tokens_for_user(self.user_id)
title, subtitle, body = self.title_and_body(story, usersub)
image_url = None
if len(story['image_urls']):
image_url = story['image_urls'][0]
# print image_url
for token in tokens.ios_tokens:
logging.user(
user, '~BMStory notification by iOS: ~FY~SB%s~SN~BM~FY/~SB%s' %
(story['story_title'][:50], usersub.feed.feed_title[:50]))
payload = Payload(alert={
'title': title,
'subtitle': subtitle,
'body': body
},
category="STORY_CATEGORY",
mutable_content=True,
custom={
'story_hash': story['story_hash'],
'story_feed_id': story['story_feed_id'],
'image_url': image_url,
})
apns.gateway_server.send_notification(token, payload)
def send_android(self, story):
if not self.is_android: return
def send_email(self, story, usersub):
if not self.is_email: return
feed = usersub.feed
story_content = self.sanitize_story(story['story_content'])
params = {
"story": story,
"story_content": story_content,
"feed": feed,
"feed_title": usersub.user_title or feed.feed_title,
"favicon_border": feed.favicon_color,
}
from_address = '*****@*****.**'
to_address = '%s <%s>' % (usersub.user.username, usersub.user.email)
text = render_to_string('mail/email_story_notification.txt', params)
html = render_to_string('mail/email_story_notification.xhtml', params)
subject = '%s: %s' % (usersub.user_title
or usersub.feed.feed_title, story['story_title'])
subject = subject.replace('\n', ' ')
msg = EmailMultiAlternatives(subject,
text,
from_email='NewsBlur <%s>' % from_address,
to=[to_address])
msg.attach_alternative(html, "text/html")
try:
msg.send()
except boto.ses.connection.ResponseError, e:
logging.user(usersub.user,
'~BMStory notification by email error: ~FR%s' % e)
logging.user(
usersub.user,
'~BMStory notification by email: ~FY~SB%s~SN~BM~FY/~SB%s' %
(story['story_title'][:50], usersub.feed.feed_title[:50]))
class MStatistics(mongo.Document):
key = mongo.StringField(unique=True)
value = mongo.StringField()
meta = {
'collection': 'statistics',
'allow_inheritance': False,
'indexes': ['key'],
}
def __unicode__(self):
return "%s: %s" % (self.key, self.value)
@classmethod
def all(cls):
values = dict([(stat.key, stat.value) for stat in cls.objects.all()])
for key, value in values.items():
if key in ('avg_time_taken', 'sites_loaded'):
values[key] = json.decode(value)
elif key in ('feeds_fetched', 'premium_users', 'standard_users',
'latest_sites_loaded', 'max_sites_loaded'):
values[key] = int(value)
elif key in ('latest_avg_time_taken', 'max_avg_time_taken'):
values[key] = float(value)
return values
@classmethod
def collect_statistics(cls):
last_day = datetime.datetime.now() - datetime.timedelta(hours=24)
cls.collect_statistics_feeds_fetched(last_day)
cls.collect_statistics_premium_users(last_day)
cls.collect_statistics_standard_users(last_day)
cls.collect_statistics_sites_loaded(last_day)
@classmethod
def collect_statistics_feeds_fetched(cls, last_day=None):
if not last_day:
last_day = datetime.datetime.now() - datetime.timedelta(hours=24)
feeds_fetched = MFeedFetchHistory.objects(
fetch_date__gte=last_day).count()
cls.objects(key='feeds_fetched').update_one(upsert=True,
key='feeds_fetched',
value=feeds_fetched)
return feeds_fetched
@classmethod
def collect_statistics_premium_users(cls, last_day=None):
if not last_day:
last_day = datetime.datetime.now() - datetime.timedelta(hours=24)
premium_users = Profile.objects.filter(last_seen_on__gte=last_day,
is_premium=True).count()
cls.objects(key='premium_users').update_one(upsert=True,
key='premium_users',
value=premium_users)
return premium_users
@classmethod
def collect_statistics_standard_users(cls, last_day=None):
if not last_day:
last_day = datetime.datetime.now() - datetime.timedelta(hours=24)
standard_users = Profile.objects.filter(last_seen_on__gte=last_day,
is_premium=False).count()
cls.objects(key='standard_users').update_one(upsert=True,
key='standard_users',
value=standard_users)
return standard_users
@classmethod
def collect_statistics_sites_loaded(cls, last_day=None):
if not last_day:
last_day = datetime.datetime.now() - datetime.timedelta(hours=24)
now = datetime.datetime.now()
sites_loaded = []
avg_time_taken = []
for hour in range(24):
start_hours_ago = now - datetime.timedelta(hours=hour)
end_hours_ago = now - datetime.timedelta(hours=hour + 1)
aggregates = dict(count=Count('loadtime'), avg=Avg('loadtime'))
load_times = FeedLoadtime.objects.filter(
date_accessed__lte=start_hours_ago,
date_accessed__gte=end_hours_ago).aggregate(**aggregates)
sites_loaded.append(load_times['count'] or 0)
avg_time_taken.append(load_times['avg'] or 0)
sites_loaded.reverse()
avg_time_taken.reverse()
values = (
('sites_loaded', json.encode(sites_loaded)),
('avg_time_taken', json.encode(avg_time_taken)),
('latest_sites_loaded', sites_loaded[-1]),
('latest_avg_time_taken', avg_time_taken[-1]),
('max_sites_loaded', max(sites_loaded)),
('max_avg_time_taken', max(1, max(avg_time_taken))),
)
for key, value in values:
cls.objects(key=key).update_one(upsert=True, key=key, value=value)
class FakeModelDB(stormbase.StormBaseDB):
state = me.StringField(required=True)
class ExclusionModel(gj.Document):
to_json_exclude = db.StringField(exclude_to_json=True)
from_json_exclude = db.IntField(exclude_from_json=True)
json_exclude = db.StringField(exclude_json=True)
required = db.StringField(required=True)
class Messages(me.Document):
meta = {'collection': 'questions'}
timestamp = me.DecimalField(precision=6)
user = me.StringField()
data = me.DictField()
