class Content(Model):
"""Ancestor class for curated content objects: Theme, Topic, Lesson, and
Practice.
"""
name = sndb.StringProperty(required=True, search_type=search.TextField)
summary = sndb.TextProperty(default='', search_type=search.TextField)
tags = sndb.StringProperty(repeated=True, search_type=search.AtomField)
subjects = sndb.StringProperty(repeated=True, search_type=search.AtomField)
min_grade = sndb.IntegerProperty(default=0, search_type=search.NumberField)
max_grade = sndb.IntegerProperty(default=13,
search_type=search.NumberField)
promoted = sndb.BooleanProperty(default=False,
search_type=search.AtomField)
class Vote(Model):
"""Always in a group under a User."""
vote_for = sndb.BooleanProperty(default=True)
practice_id = sndb.StringProperty(default=None)
lesson_id = sndb.StringProperty(default=None)
book_id = sndb.StringProperty(default=None)
page_id = sndb.StringProperty(default=None)
entity_id = sndb.StringProperty(default=None)
entity_type = sndb.StringProperty(default=None)
@classmethod
def create(klass, **kwargs):
# if new dynamic entity fields aren't used (deprecated code), create them for now.
if 'entity_type' in kwargs:
target_entity_type = kwargs['entity_type']
# TODO: fix this, shouldn't need this fallback for prod.
else:
simple_whitelist = ['practice', 'lesson', 'book', 'page']
for whitelist_type in simple_whitelist:
id_prop = '{}_id'.format(whitelist_type)
if id_prop in kwargs:
target_entity_type = whitelist_type
# Get reference for model object from imports based on entity type
target_model = globals()[target_entity_type.capitalize()]
# Construct expected id property from entity type
target_id_property = '{}_id'.format(target_entity_type)
existing_uid = kwargs[target_id_property]
# Replace entity_id with full version
kwargs[target_id_property] = target_model.get_long_uid(existing_uid)
vote = super(klass, klass).create(**kwargs)
# Increment votes_for on entity if success
if hasattr(vote, target_id_property):
vote_entity_id = getattr(vote, target_id_property)
entity = target_model.get_by_id(vote_entity_id)
if entity is not None:
entity.votes_for += 1
entity.put()
return vote
def parent_user_id(self):
return self.key.parent().id()
class Vote(Model):
"""Always in a group under a User."""
vote_for = sndb.BooleanProperty(default=True)
practice_id = sndb.StringProperty(default=None)
lesson_id = sndb.StringProperty(default=None)
@classmethod
def create(klass, **kwargs):
if ('practice_id' not in kwargs) and ('lesson_id' not in kwargs):
raise Exception(
'Must specify either a practice or a lesson when '
'creating a vote. Received kwargs: {}'.format(kwargs))
# Check if user has voted on this entity
# existing_vote = klass.query(klass.deleted == False, ancestor=**kwargs['parent'].key, )
# if existing_vote:
# raise Exception('Found an existing vote on this entity for '
# 'current user. Received kwargs: {}'
# .format(kwargs))
# Replace lesson_id and practice_id with full versions
if 'practice_id' in kwargs:
kwargs[u'practice_id'] = Practice.get_long_uid(
kwargs[u'practice_id'])
if 'lesson_id' in kwargs:
kwargs[u'lesson_id'] = Lesson.get_long_uid(kwargs[u'lesson_id'])
vote = super(klass, klass).create(**kwargs)
# Increment votes_for on Practice or Lesson if success
if vote.practice_id:
practice = Practice.get_by_id(vote.practice_id)
if practice is not None:
practice.votes_for += 1
practice.put()
if vote.lesson_id:
lesson = Lesson.get_by_id(vote.lesson_id)
if lesson is not None:
lesson.votes_for += 1
lesson.put()
return vote
def parent_user_id(self):
return self.key.parent().id()
class Assessment(Model):
"""A MSK representation of a mindset meter assessment, created by PERTS.
Mostly just a name to list avaiable assessments and to organize surveys
created from them. The bulk of the definition of an assessment happens on
[mindsetmeter](survey.perts.net) in the form of html templates for surveys
and reports.
Always available to everyone, so listed by default.
"""
name = sndb.StringProperty(required=True, search_type=search.TextField)
# Defined on the client, but typically: self.name.lower().replace(' ', '-')
# So for 'Growth Mindset' this is 'growth-mindset', and it's used to make
# URLs for mindset meter, so users would be directed to
# 'survey.perts.net/take/growth-mindset'
url_name = sndb.StringProperty(required=True, validator=url_name_validator)
description = sndb.TextProperty(default='', search_type=search.TextField)
num_phases = sndb.IntegerProperty(required=True)
listed = sndb.BooleanProperty(default=True)
class Email(Model):
"""An email in a queue (not necessarily one that has been sent).
Emails have regular fields, to, from, subject, body. Also a send date and
an sent boolean to support queuing.
Uses jinja to attempt to attempt to interpolate values in the
template_data dictionary into the body and subject.
Body text comes in two flavors: raw text and html. The raw text is
interpreted as markdown (after jinja processing) to auto-generate the html
version (templates/email.html is also used for default styles). Which
version users see depends on their email client.
Consequently, all emails should be composed in markdown.
"""
to_address = sndb.StringProperty(required=True)
from_address = sndb.StringProperty(
default=config.from_server_email_address)
reply_to = sndb.StringProperty(default=config.from_server_email_address)
subject = sndb.StringProperty(default="A message from the Mindset Kit")
body = sndb.TextProperty()
html = sndb.TextProperty()
template = sndb.StringProperty()
template_data_string = sndb.TextProperty(default='{}')
scheduled_date = sndb.DateProperty(auto_now_add=True)
was_sent = sndb.BooleanProperty(default=False)
was_attempted = sndb.BooleanProperty(default=False)
errors = sndb.TextProperty()
@property
def template_data(self):
return json.loads(self.template_data_string)
@template_data.setter
def template_data(self, value):
self.template_data_string = json.dumps(value,
default=util.json_dumps_default)
return value
@classmethod
def create(klass, **kwargs):
"""
WARNING: Do NOT pass any objects in kwargs['template_data']
And be sure any associated templates use plain values in jinja code
"""
email = super(klass, klass).create(**kwargs)
return email
@classmethod
def we_are_spamming(self, email):
"""Did we recently send email to this recipient?
@todo discuss - I can think of cases where this can be harmful...
IE 2 different users comment on 2 different uploads from same user
I guess we should introduce Notifications in the app :)
"""
to = email.to_address
# We can spam admins, like
# so we white list them in the config file
if to in config.addresses_we_can_spam:
return False
# We can also spam admins living at a @mindsetkit.org
if to.endswith('mindsetkit.org'):
return False
since = datetime.datetime.utcnow() - datetime.timedelta(
minutes=config.suggested_delay_between_emails)
query = Email.query(
Email.was_sent == True,
Email.scheduled_date >= since,
Email.to_address == to,
)
return query.count(limit=1) > 0
@classmethod
def send(self, emails):
to_addresses = []
for email in emails:
if self.we_are_spamming(email):
# Do not send
# This user has already recieved very recent emails
# Debugging info
logging.error("We are spamming {}:\n{}".format(
email.to_address, email.to_dict()))
elif email.to_address in to_addresses:
# Do not send
# We don't send multiple emails to an address per 'send'
# Debugging info
logging.error("We are spamming {}:\n{}".format(
email.to_address, email.to_dict()))
else:
# Not spam! Let's send it
to_addresses.append(email.to_address)
# Debugging info
logging.info(u"sending email: {}".format(email.to_dict()))
logging.info(u"to: {}".format(email.to_address))
logging.info(u"subject: {}".format(email.subject))
if email.body:
logging.info(u"body:\n{}".format(email.body))
mandrill.send(
to_address=email.to_address,
subject=email.subject,
body=email.body,
template_data=email.template_data,
)
elif email.template:
logging.info(u"template: {}".format(email.template))
mandrill.send(
to_address=email.to_address,
subject=email.subject,
template=email.template,
template_data=email.template_data,
)
email.was_sent = True
logging.info("""Sent successfully!""")
# Note that we are attempting to send so that we don't keep attempting.
email.was_attempted = True
email.put()
@classmethod
def fetch_pending_emails(self):
to_send = Email.query(
Email.deleted == False,
Email.scheduled_date <= datetime.datetime.utcnow(),
Email.was_sent == False,
Email.was_attempted == False,
)
return to_send.fetch()
@classmethod
def send_pending_email(self):
"""Send the next unsent emails in the queue.
"""
emails = self.fetch_pending_emails()
if emails:
self.send(emails)
return emails
else:
return None
class Model(ndb.Model):
"""Superclass for all others; contains generic properties and methods."""
# This id is an encoding of the entity's key. For root entities (those
# with no ancestors), it's a class name (same as a GAE "kind") and
# an identifier separated by an underscore.
# Example: Theme_mW4iQ4cO
# For entities in a group (those with ancestors), their full heirarchy is
# encoded, ending with the root entity, separated by periods.
# Example: Comment_p46aOHS6.User_80h41Q4c
# An id is always sufficient to look up an entity directly.
# N.B.: We can't just call it 'id' because that breaks ndb.
uid = sndb.TextProperty(default=None)
short_uid = sndb.TextProperty(default=None, search_type=search.AtomField)
deleted = sndb.BooleanProperty(default=False)
created = sndb.DateTimeProperty(auto_now_add=True,
search_type=search.DateField)
modified = sndb.DateTimeProperty(auto_now=True,
search_type=search.DateField)
listed = sndb.BooleanProperty(default=False)
# Do NOT use sndb.JsonProperty here! That stores the data as a blob, which
# is DROPPED upon export to BigQuery. Instead, this stores the JSON as a
# string, which you can cleanly asscess as a python object through the
# `json_properties` property, below.
json_properties_string = sndb.TextProperty(default='{}')
@property
def json_properties(self):
return json.loads(self.json_properties_string)
@json_properties.setter
def json_properties(self, value):
self.json_properties_string = json.dumps(
value, default=util.json_dumps_default)
return value
@classmethod
def create(klass, **kwargs):
# ndb expects parents to be set with a key, but we find it more
# convenient to pass in entities. Do the translation here.
if 'parent' in kwargs:
parent = kwargs['parent'] # used in id generation
kwargs['parent'] = kwargs['parent'].key
else:
parent = None
if 'id' in kwargs:
# User has supplied their own id. That's okay (it makes certain
# URLs nice and readable), but it's not a real uid yet b/c it
# doesn't adhere to our ClassName_identifierXYZ convention. We'll
# pass it into generated_uid() later.
identifier = kwargs['id']
del kwargs['id']
else:
identifier = None
# ndb won't recognize the simulated json_properties if passed directly
# as a keyword argument. Translate that too.
if 'json_properties' in kwargs:
kwargs['json_properties_string'] = json.dumps(
kwargs['json_properties'])
del kwargs['json_properties']
# ndb won't recognize the simulated template_data if passed directly
# as a keyword argument. Translate that too.
if 'template_data' in kwargs:
kwargs['template_data_string'] = json.dumps(
kwargs['template_data'])
del kwargs['template_data']
# Make sure id is unique, otherwise "creating" this entity will
# overwrite an existing one, which could be a VERY hard bug to chase
# down.
for x in range(5):
uid = klass.generate_uid(parent, identifier)
existing_entity = Model.get_by_id(uid)
if not existing_entity:
break
if existing_entity:
if identifier:
raise Exception("Entity {} already exists.".format(uid))
else:
raise Exception("After five tries, could not generate a "
"unique id. This should NEVER happen.")
kwargs['uid'] = uid
kwargs['short_uid'] = klass.convert_uid(uid)
return klass(id=uid, **kwargs)
@classmethod
def generate_uid(klass, parent=None, identifier=None, existing=False):
"""Make a gobally unique id string, e.g. 'Program_mW4iQ4cO'.
Using 8 random chars, if we made 10,000 entities of the same kind (and
comments by a single user count as a kind), the probability of
duplication is 2E-7. Combined with the five attempts at uniqueness
in Model.create(), chances of duplication are essentially nil.
http://en.wikipedia.org/wiki/Universally_unique_identifier#Random_UUID_probability_of_duplicates
If a parent entity is specified, it becomes part of the uid, like this:
Comment_p46aOHS6.User_80h41Q4c
If an identifier is specified, it is used instead of random characters:
Theme_growth-mindset-is-cool
"""
if identifier:
if not re.match(r'^[A-Za-z0-9\-]+$', identifier):
raise Exception("Invalid identifier: {}. Letters, numbers, "
"and hyphens only.".format(identifier))
suffix = identifier
else:
chars = (string.ascii_uppercase + string.ascii_lowercase +
string.digits)
suffix = ''.join(random.choice(chars) for x in range(8))
uid = klass.__name__ + '_' + suffix
# Because comments exist as descendants of other entities, a simple
# id-as-key-name is insufficient. We must store information about its
# ancestry as well. Example:
# Comment_p46aOHS6.User_80h41Q4c
if parent:
uid += '.' + parent.uid
return uid
@classmethod
def convert_uid(klass, short_or_long_uid):
"""Changes long-form uid's to short ones, and vice versa.
Long form example: Theme_growth-mindset-is-cool
Short form exmaple: growth-mindset-is-cool
"""
if '_' in short_or_long_uid:
return short_or_long_uid.split('_')[1]
else:
return klass.generate_uid(identifier=short_or_long_uid)
@classmethod
def get_long_uid(klass, short_or_long_uid):
"""Changes short or long-form uid's to long ones.
Long form example: Theme_growth-mindset-is-cool
Short form exmaple: growth-mindset-is-cool
"""
if '_' in short_or_long_uid:
return short_or_long_uid
else:
# Check the characters to make sure this could be valid
if re.match(r'^[A-Za-z0-9\-]+$', short_or_long_uid):
return klass.generate_uid(identifier=short_or_long_uid)
else:
# Original UID is invalid anyway, return None
None
@classmethod
def get_parent_uid(klass, uid):
"""Don't use the datastore; get parent ids based on convention."""
if '.' not in uid:
raise Exception("Can't get parent of id: {}".format(uid))
return '.'.join(uid.split('.')[1:])
@classmethod
def get_kind(klass, obj):
"""Get the kind (class name string) of an entity, key, or id.
Examples:
* For a Theme entity, the kind is 'Theme'
* For the id 'Comment_p46aOHS6.User_80h41Q4c',
the kind is 'Comment'.
"""
if isinstance(obj, basestring):
return str(obj.split('_')[0])
elif isinstance(obj, Model):
return obj.__class__.__name__
elif isinstance(obj, ndb.Key):
return obj.kind()
else:
raise Exception('Model.get_kind() invalid input: {} ({}).'.format(
obj, str(type(obj))))
@classmethod
def get_class(klass, kind):
"""Convert a class name string (same as a GAE kind) to a class.
See http://stackoverflow.com/questions/1176136/convert-string-to-python-class-object
"""
return getattr(sys.modules['model'], kind, None)
@classmethod
def id_to_key(klass, id):
parts = id.split('.')
pairs = [(Model.get_kind(p), '.'.join(parts[-x:]))
for x, p in enumerate(parts)]
return ndb.Key(pairs=reversed(pairs))
@classmethod
def get_by_id(klass, id_or_list):
"""The main way to get entities with known ids.
Args:
id_or_list: A single perts id string, or a list of such strings,
of any kind or mixed kinds.
Returns an entity or list of entities, depending on input.
"""
# Sanitize input to a list of strings.
if type(id_or_list) in [str, unicode]:
ids = [id_or_list]
elif type(id_or_list) is list:
ids = id_or_list
else:
# I don't think we should be blocking code here
# Problem was occuring when you search a bad id or just None
# Ex. "/topics/foobar."
return None
raise Exception("Invalid id / id: {}.".format(id_or_list))
# Iterate through the list, generating a key for each id
keys = []
for id in ids:
keys.append(Model.id_to_key(id))
results = ndb.get_multi(keys)
# Wrangle results into expected structure.
if len(results) is 0:
return None
if type(id_or_list) in [str, unicode]:
return results[0]
if type(id_or_list) is list:
return results
def __str__(self):
"""A string represenation of the entity. Goal is to be readable.
Returns, e.g. <id_model.User User_oha4tp8a>.
Native implementation does nothing useful.
"""
return '<{}>'.format(self.key.id())
def __repr__(self):
"""A unique representation of the entity. Goal is to be unambiguous.
But our ids are unambiguous, so we can just forward to __str__.
Native implemention returns a useless memory address, e.g.
<id_model.User 0xa5e418cdd>
The big benefit here is to be able to print debuggable lists of
entities, without need to manipulate them first, e.g.
print [entity.id for entity in entity_list]
Now you can just write
print entity_list
"""
return self.__str__()
# special methods to allow comparing entities, even if they're different
# instances according to python
# https://groups.google.com/forum/?fromgroups=#!topic/google-appengine-python/uYneYzhIEzY
def __eq__(self, value):
"""Allows entity == entity to be True if keys match.
Is NOT called by `foo is bar`."""
if self.__class__ == value.__class__:
return self.key.id() == value.key.id()
else:
return False
# Because we defined the 'equals' method, eq, we must also be sure to
# define the 'not equals' method, ne, otherwise you might get A == B is
# True, and A != B is also True!
def __ne__(self, value):
"""Allows entity != entity to be False if keys match."""
if self.__class__ == value.__class__:
# if they're the same class, compare their keys
return self.key.id() != value.key.id()
else:
# if they're not the same class, then it's definitely True that
# they're not equal.
return True
def __hash__(self):
"""Allows entity in entity_list to be True."""
return hash(str(self.key))
def _post_put_hook(self, future):
"""Executes after an entity is put.
1. Updates search index
To allow for batch processing (i.e. doing the stuff this function does
for many entities all at once, instead of doing it here one by one),
this function can be disabled by adding an emphemeral attribute (one
not saved to the datastore) to the entity. Example:
# Don't do post-put processing for these entities, so we can handle
# them in a batch later.
for e in entities:
e.forbid_post_put_hook = True
ndb.put_multi(entities)
"""
if getattr(self, 'forbid_post_put_hook', False):
return
if self.get_kind(self) in config.indexed_models:
index = search.Index(config.content_index)
if self.listed:
logging.info("Indexing content for search.")
index.put(self.to_search_document())
else:
# Unlisted entities should be actively removed from search.
logging.info("Removing unlisted content from search.")
index.delete(self.uid)
@classmethod
def _post_delete_hook(klass, key, future):
"""We rarely truely delete entities, but when we do, we prefer Dos
Equis. I mean, we want to delete them from the search index."""
if klass.get_kind(key) in config.indexed_models:
logging.info(
"Removing hard-deleted content from search: {}".format(
key.id()))
search.Index(config.content_index).delete(key.id())
def to_client_dict(self, override=None):
"""Convert an app engine entity to a dictionary.
Ndb provides a to_dict() method, but we want to add creature-features:
1. Put properties in a predictable order so they're easier to read.
2. Remove or mask certain properties based on the preferences of our
javascript.
3. Handle our string-based json_properties correctly.
4. Ndb (different from db) stores datetimes as true python datetimes,
which JSON.dumps() doesn't know how to handle. We'll convert them
to ISO strings (e.g. "2010-04-20T20:08:21.634121")
Args:
override: obj, if provided, method turns this object into
a dictionary, rather than self.
"""
output = self.to_dict()
output['json_properties'] = self.json_properties
del output['json_properties_string']
for k, v in output.items():
if isinstance(v, dict):
for k2, v2 in output.items():
if hasattr(v2, 'isoformat'):
output[k][k2] = v2.isoformat()
if hasattr(v, 'isoformat'):
output[k] = v.isoformat()
client_safe_output = {}
for k, v in output.items():
if k in config.client_private_properties:
client_safe_output['_' + k] = v
elif k not in config.client_hidden_properties:
client_safe_output[k] = v
# order them so they're easier to read
ordered_dict = collections.OrderedDict(
sorted(client_safe_output.items(), key=lambda t: t[0]))
return ordered_dict
def _get_search_fields(self):
fields = []
klass = self.__class__
# Scan the properties in entity's class definition for information on
# how the entity should be converted to a search document. Properties
# can have search field types defined; we use those to construct the
# document.
for prop_name in dir(klass):
# This is the abstract property object in the class definition.
# E.g. sndb.StringProperty()
prop = getattr(klass, prop_name)
# This is the actual data defined on the entity
value = getattr(self, prop_name)
# This is (maybe) the type of search field the property should be
# converted to.
search_type = getattr(prop, 'search_type', None)
# The dir() function iterates all object attributes; only deal with
# those which 1) are datastore properties, 2) aren't private, and
# 3) have a search type defined.
is_field = (isinstance(prop, ndb.model.Property)
and not prop_name.startswith('_') and search_type)
if is_field:
# Search documents field names aren't unique; storing a list of
# values means making many fields of the same name. For
# brevity, make everything as a (possibly single-element) list.
if not type(value) is list:
value = [value]
for v in value:
# It will probably be common to put a boolean in a search
# document. The easiest way is to make it a string in an
# atom field.
if type(v) is bool and search_type is search.AtomField:
v = 'true' if v else 'false'
fields.append(search_type(name=prop_name, value=v))
# kind is not a field, but it's useful as a filter.
fields.append(search.AtomField(name='kind',
value=Model.get_kind(self)))
return fields
def to_search_document(self, rank=None):
fields = self._get_search_fields()
# Our uid's are already globally unique, so it's fine (as well as
# extremely convenient) to use them as search document ids as well.
# N.B. english language hard coded here. If we ever internationalize,
# will need to be dynamic.
return search.Document(doc_id=self.uid,
fields=fields,
rank=rank,
language='en')
class Comment(Model):
"""Always in a group under a User."""
body = sndb.TextProperty(default='')
practice_id = sndb.StringProperty(default=None)
lesson_id = sndb.StringProperty(default=None)
page_id = sndb.StringProperty(default=None)
# Overrides Model's default for this property, which is False. We always
# want to see comments.
listed = sndb.BooleanProperty(default=True)
@classmethod
def create(klass, **kwargs):
if ('practice_id' not in kwargs) and ('lesson_id' not in kwargs) and (
'page_id' not in kwargs):
raise Exception(
'Must specify a practice, lesson, or page when '
'creating a comment. Received kwargs: {}'.format(kwargs))
comment = super(klass, klass).create(**kwargs)
# For email notifications
content_url = '/'
content = None
if comment.page_id:
page = Page.get_by_id(comment.page_id)
if page is not None:
page.num_comments += 1
page.put()
# For email
content = page
content_url = '/pages/{}'.format(page.short_uid)
# Send email to creator
author_ids = page.authors
commenter = comment.get_parent_user()
authors = User.get_by_id(author_ids) or []
for author in authors:
# logic to not email yourself...
if author != commenter.email:
short_name = author.first_name or ''
full_name = author.full_name
commenter_image_url = commenter.profile_image
# Uses Email model to queue email and prevent spam
email = Email.create(
to_address=author.email,
subject=
"Someone commented on your BELE Library upload",
template="comment_creator_notification.html",
template_data={
'short_name':
short_name,
'full_name':
full_name,
'commenter_name':
commenter.full_name,
'commenter_image_url':
commenter_image_url,
'content_name':
content.title,
'comment_body':
comment.body,
'content_url':
content_url,
'domain':
'https://{}'.format(
os.environ['HOSTING_DOMAIN']),
'year':
datetime.date.today().year,
},
)
email.put()
# Send email to any users @replied to
usernames = re.search('\@(\w+)', comment.body)
if usernames is not None:
username = usernames.group(0).split('@')[1]
# Fetch user from username and send email message
replied_to = User.query(
User.username == username).fetch(1)
if replied_to:
replied_to = replied_to[0]
short_name = replied_to.first_name or ''
full_name = replied_to.full_name
commenter_image_url = commenter.profile_image
# Uses Email model to queue email and prevent spam
email = Email.create(
to_address=replied_to.email,
subject=
"Someone replied to you on BELE Library",
template="comment_reply_notification.html",
template_data={
'short_name':
short_name,
'full_name':
full_name,
'commenter_name':
commenter.full_name,
'commenter_image_url':
commenter_image_url,
'content_name':
content.title,
'comment_body':
comment.body,
'content_url':
content_url,
'domain':
'https://{}'.format(
os.environ['HOSTING_DOMAIN']),
'year':
datetime.date.today().year,
},
)
email.put()
# Email interested team members that a comment has been created
mandrill.send(
to_address=config.comment_recipients,
subject="New Comment on BELE Library!",
template="comment_notification.html",
template_data={
'comment': comment,
'user': comment.get_parent_user(),
'content_name': content.title,
'content_url': content_url,
'domain': 'https://{}'.format(os.environ['HOSTING_DOMAIN'])
},
)
logging.info('model.Comment queueing an email to: {}'.format(
config.comment_recipients))
return comment
@classmethod
def convert_uid(klass, short_or_long_uid):
"""Changes long-form uid's to short ones, and vice versa.
Overrides method provided in Model.
Long form example: Practice_Pb4g9gus.User_oha4tp8a
Short form exmaple: Pb4g9gusoha4tp8a
"""
if '.' in short_or_long_uid:
parts = short_or_long_uid.split('.')
return ''.join([x.split('_')[1] for x in parts])
else:
return 'Comment_{}.User_{}'.format(short_or_long_uid[:8],
short_or_long_uid[8:])
def parent_user_id(self):
return self.key.parent().id()
def get_parent_user(self):
return self.key.parent().get()
class Practice(Content):
"""Always in a group under a User."""
mindset_tags = sndb.StringProperty(repeated=True)
practice_tags = sndb.StringProperty(repeated=True)
time_of_year = sndb.StringProperty(default='')
class_period = sndb.StringProperty(default='')
# Stores the UID of 'associated' content (Theme or Topic)
# Used to fetch related practices from various pages
associated_content = sndb.StringProperty(default='')
type = sndb.StringProperty(default='text', search_type=search.AtomField)
body = sndb.TextProperty(default='', search_type=search.TextField)
youtube_id = sndb.StringProperty(default='')
iframe_src = sndb.StringProperty(default='')
has_files = sndb.BooleanProperty(default=False,
search_type=search.AtomField)
pending = sndb.BooleanProperty(default=True)
votes_for = sndb.IntegerProperty(default=0, search_type=search.NumberField)
num_comments = sndb.IntegerProperty(default=0,
search_type=search.NumberField)
@classmethod
def create(klass, **kwargs):
"""Sends email to interested parties.
"""
practice = super(klass, klass).create(**kwargs)
# Email interested parties that a practice has been uploaded.
mandrill.send(
to_address=config.practice_upload_recipients,
subject="Practice Uploaded to Mindset Kit!",
template="practice_upload_notification.html",
template_data={
'user': practice.get_parent_user(),
'practice': practice,
'domain': os.environ['HOSTING_DOMAIN']
},
)
logging.info('model.Practice queueing an email to: {}'.format(
config.practice_upload_recipients))
return practice
@classmethod
def convert_uid(klass, short_or_long_uid):
"""Changes long-form uid's to short ones, and vice versa.
Overrides method provided in Model.
Long form example: Practice_Pb4g9gus.User_oha4tp8a
Short form exmaple: Pb4g9gusoha4tp8a
"""
if '.' in short_or_long_uid:
parts = short_or_long_uid.split('.')
return ''.join([x.split('_')[1] for x in parts])
else:
return 'Practice_{}.User_{}'.format(short_or_long_uid[:8],
short_or_long_uid[8:])
@classmethod
def get_long_uid(klass, short_or_long_uid):
"""Changes short of long-form uid's to long ones.
Overrides method provided in Model.
Long form example: Practice_Pb4g9gus.User_oha4tp8a
Short form exmaple: Pb4g9gusoha4tp8a
"""
if '.' in short_or_long_uid: # is long
return short_or_long_uid
else: # is short
return 'Practice_{}.User_{}'.format(short_or_long_uid[:8],
short_or_long_uid[8:])
@classmethod
def get_related_practices(klass, content, count):
"""Fetches practices related to a content object
Will default to random pratices if none found.
"""
related_practices = []
query = Practice.query(
Practice.deleted == False,
Practice.listed == True,
)
if content.uid:
if Model.get_kind(content.uid) == 'Practice':
if content.associated_content:
query = query.filter(Practice.associated_content ==
content.associated_content)
query = query.filter(Practice.uid != content.uid)
else:
query = query.filter(
Practice.associated_content == content.uid)
query.order(-Practice.created)
# Pull a 'bucket' of practices to sample from
related_practices_bucket = query.fetch(15)
# Only return a random selection of the practices
if len(related_practices_bucket) > count:
related_practices = random.sample(related_practices_bucket, count)
elif len(related_practices_bucket) <= 0:
related_practices = []
else:
related_practices = related_practices_bucket
return related_practices
@classmethod
def get_popular_practices(klass):
"""Fetches popular practices to display on landing page
@todo: figure out a way to generate this list
- Possibly adding a field to practices and flagging x practices / week
- Needs more discussion
"""
practices = []
query = Practice.query(
Practice.deleted == False,
Practice.listed == True,
Practice.promoted == True,
)
query.order(-Practice.created)
practices = query.fetch(20)
if len(practices) > 6:
practices = random.sample(practices, 6)
return practices
def add_file_data(self, file_dicts):
"""Save dictionaries of uploaded file meta data."""
jp = self.json_properties
if 'files' not in jp:
jp['files'] = []
jp['files'].extend(file_dicts)
self.json_properties = jp
self.has_files = len(jp['files']) > 0
def remove_file_data(self, file_key):
"""Remove file dictionaries from existing json_properties"""
jp = self.json_properties
# Find and remove file from 'files' in json_properties
if 'files' in jp:
for index, file_dict in enumerate(jp['files']):
if file_key == file_dict[u'gs_object_name']:
jp['files'].pop(index)
self.json_properties = jp
self.has_files = len(jp['files']) > 0
def get_parent_user(self):
return self.key.parent().get()
def check_status_update(self, **kwargs):
"""Checks the status of an updated practice to determine if the creator
should be notified of approval or rejection
Only triggered if pending set from True to False (prevents duplicates)
"""
if (self.pending is True and kwargs.get('pending') is False):
creator = self.get_parent_user()
short_name = creator.first_name if creator.first_name else ''
full_name = creator.full_name
if (self.listed is False and kwargs.get('listed') is True):
# Send acceptance message
# @todo: add name to subject line
mandrill.send(
to_address=creator.email,
subject="Your practice upload is approved!",
template="accepted_notification.html",
template_data={
'short_name': short_name,
'full_name': full_name,
'practice_name': self.name,
'practice_url': '/practices/' + self.short_uid,
'domain': os.environ['HOSTING_DOMAIN']
},
)
else:
# Send rejection message
mandrill.send(
to_address=creator.email,
subject="We couldn't approve your practice...",
template="rejected_notification.html",
template_data={
'short_name': short_name,
'full_name': full_name,
'practice_name': self.name,
'practice_url': '/practices/' + self.short_uid,
'edit_url': '/practices/edit/' + self.short_uid,
'domain': os.environ['HOSTING_DOMAIN']
},
)
def to_search_document(self, rank=None):
"""Extends inherited method in Model."""
fields = super(Practice, self)._get_search_fields()
# Add information about the parent user to the search document.
user = self.get_parent_user()
# Allow for empty first/last names, and default to an empty string.
if user is not None:
user_name = ''.join([(user.first_name or ''), (user.last_name
or '')])
fields.append(search.TextField(name='author', value=user_name))
# Simplify checking for video and file attachments
if self.has_files:
fields.append(search.AtomField(name='content_type', value='files'))
if self.youtube_id != '':
fields.append(search.AtomField(name='content_type', value='video'))
return search.Document(doc_id=self.uid,
fields=fields,
rank=rank,
language='en')
class User(Model):
"""Serve as root entities for Comments and Practices."""
# see config.auth_types
last_login = sndb.DateTimeProperty(auto_now_add=True)
auth_id = sndb.StringProperty(required=True,
validator=lambda prop, value: value.lower())
facebook_id = sndb.StringProperty(default=None)
google_id = sndb.StringProperty(default=None)
hashed_password = sndb.StringProperty(default=None)
first_name = sndb.StringProperty(default='')
last_name = sndb.StringProperty(default='')
email = sndb.StringProperty(required=True,
validator=lambda prop, value: value.lower())
is_admin = sndb.BooleanProperty(default=False)
receives_updates = sndb.BooleanProperty(default=True)
image_url = sndb.StringProperty(default='')
short_bio = sndb.StringProperty(default='')
# Username for display
username = sndb.StringProperty()
# Username for uniqueness checking
canonical_username = sndb.ComputedProperty(
lambda self: self.username.lower() if self.username else '')
@classmethod
def create(klass, check_uniqueness=True, **kwargs):
"""Checks email uniqueness before creating; raises DuplicateUser.
Checking uniqueness always generates a Unique entity (an entity of
class 'Unique') that permanently reserves the email address. But
checking is also optional, and can be passed as False for testing.
See http://webapp-improved.appspot.com/_modules/webapp2_extras/appengine/auth/models.html#User.create_user
"""
# If no username, generate one!
if 'username' not in kwargs:
kwargs['username'] = User.create_username(**kwargs)
else:
if User.is_valid_username(kwargs['username']):
raise InvalidUsername(
"Invalid username {}. Use only letters, numbers, dashes, and underscores."
.format(kwargs['username']))
# Check for uniqueness of email and username
if check_uniqueness:
uniqueness_key = 'User.email:' + kwargs['email']
is_unique_email = Unique.create(uniqueness_key)
if not is_unique_email:
raise DuplicateUser(
"There is already a user with email {}.".format(
kwargs['email']))
uniqueness_key = 'User.username:'******'username'].lower()
is_unique_username = Unique.create(uniqueness_key)
if not is_unique_username:
# Need to also delete the unused email!
# To ensure that this works on retry
is_unique_email.delete()
raise DuplicateUser(
"There is already a user with username {}.".format(
kwargs['username']))
# Register user for the mailchimp list
should_subscribe = kwargs.pop('should_subscribe', False)
if should_subscribe is not False:
fields = {}
if 'first_name' in kwargs:
fields['first_name'] = kwargs['first_name']
if 'last_name' in kwargs:
fields['last_name'] = kwargs['last_name']
subscribed = mailchimp.subscribe(kwargs['email'], **fields)
if subscribed:
kwargs['receives_updates'] = True
return super(klass, klass).create(**kwargs)
@classmethod
def get_auth_id(klass, auth_type, third_party_id):
if auth_type not in config.auth_types:
raise Exception("Invalid auth type: {}.".format(auth_type))
return '{}:{}'.format(auth_type, third_party_id)
@property
def auth_type(self):
return self.auth_id.split(':')[0]
@property
def full_name(self):
if self.first_name and self.last_name:
return self.first_name + ' ' + self.last_name
else:
return self.username
@property
def profile_image(self):
if self.image_url:
return self.image_url
elif self.facebook_id:
return 'http://graph.facebook.com/{}/picture?type=square'.format(
self.facebook_id)
else:
return 'https://www.mindsetkit.org/static/images/default-user.png'
def set_password(self, new_password):
"""May raise util.BadPassword."""
logging.info("Setting new password {} for user {}.".format(
new_password, self))
self.hashed_password = util.hash_password(new_password)
# Alert the user that their password has been changed.
mandrill.send(
to_address=self.email,
subject="Your Mindset Kit password has been changed.",
template="change_password.html",
)
logging.info('User.set_password queueing an email to: {}'.format(
self.email))
@classmethod
def check_email_uniqueness(self, email):
"""Check if email is used"""
unique = Unique.get_by_id('User.email:' + email.lower())
return unique is None
@classmethod
def check_username_uniqueness(self, username):
"""Check if username is used"""
unique = Unique.get_by_id('User.username:'******'[^A-Za-z0-9\-\_]')
return username != regex.sub('', username)
@classmethod
def create_username(klass, **kwargs):
unformatted_username = ''
if 'first_name' in kwargs and 'last_name' in kwargs:
if kwargs['last_name'] is not None:
unformatted_username = kwargs['first_name'] + kwargs[
'last_name']
else:
unformatted_username = kwargs['first_name']
elif 'email' not in kwargs:
logging.error(
u"User doesn't have an email address {}".format(kwargs))
else:
unformatted_username = kwargs['email'].split('@')[0]
regex = re.compile('[^A-Za-z0-9\-\_]')
username = regex.sub('', unformatted_username)
if not User.check_username_uniqueness(username):
# If not unique, iterate through available names
raw_username = username
for n in range(1, 1000):
username = raw_username + str(n)
if User.check_username_uniqueness(username):
break
if n == 999:
raise Exception(
"No unique username found after 1000 tries.")
return username
@classmethod
def set_subscription(self, email, should_subscribe):
"""Sets user's subscription status to Update List on mailchimp
"""
# Use 'resubscribe' and 'unsubscribe' methods
if should_subscribe is True:
subscribed = mailchimp.resubscribe(email)
# if resubscription failed, they might not be subscribed at all!
if not subscribed:
subscribed = mailchimp.subscribe(email)
elif should_subscribe is False:
subscribed = mailchimp.unsubscribe(email)
return subscribed
def update_email(self, email):
# Remove old email from unique keys so it can be re-used!
old_email = self.email
uniqueness_key = 'User.email:' + old_email
Unique.delete_multi([uniqueness_key])
# Update to new email and check uniqueness
setattr(self, 'email', email)
uniqueness_key = 'User.email:' + email
unique = Unique.create(uniqueness_key)
if not unique:
raise DuplicateField(
"There is already a user with email {}.".format(email))
# Also need to update the user in our mailchimp
mailchimp.unsubscribe(old_email)
if self.receives_updates:
subscribed = mailchimp.subscribe(email)
# if subscription failed, they might need to resubscribe!
if not subscribed:
subscribed = mailchimp.resubscribe(email)
return unique
def update_username(self, username):
if self.username is not None:
uniqueness_key = 'User.username:'******'username', username)
uniqueness_key = 'User.username:'******'User.email:' + self.email
uniqueness_key_username = '******' + self.username
Unique.delete_multi([uniqueness_key_email, uniqueness_key_username])
def add_user_image(self, blob_key):
"""Save dictionary of user image urls.
Create permenant link to user image --
You can resize and crop the image dynamically by specifying
the arguments in the URL '=sxx' where xx is an integer from 0-1600
representing the length, in pixels, of the image's longest side
Reference: https://cloud.google.com/appengine/docs/python/images/#Python_Transforming_images_from_the_Blobstore
"""
image_url = images.get_serving_url(blob_key)
self.image_url = image_url
# Create a UserImage object pointing to blobstore object
# Reference used for removal of old UserImages
user_image = UserImage.create(
user_id=self.uid,
blob_key=blob_key,
)
user_image.put()
def remove_user_image(self):
"""Removes existing user's image completely"""
self.image_url = ''
# Find associated UserImage
user_image = UserImage.query(user_id=self.uid)
# Remove permenant link for image
images.delete_serving_url(user_image.blob_key)
user_image.deleted = True
user_image.put()
class Page(Content):
####
chapters = sndb.StringProperty(repeated=True) # unordered parents
authors = sndb.StringProperty(
repeated=True, search_type=search.TextField) # unordered children
icon = sndb.StringProperty(default='', search_type=search.TextField)
iconPath = sndb.ComputedProperty(
lambda self: util.extract_value_from_json(self.icon, 'link'))
# "Challenges & Preconditions"
preconditions_for_success = sndb.TextProperty(
default=''
) # Up to 10,000 characters, HTML text with ability to include links and images.
# "Connection to Equity"
advances_equity = sndb.TextProperty(
default='') # up to 5,000 characters, HTML
# "Time Required"
time_required = sndb.StringProperty(default='') # minutes
# "Required Materials"
required_materials = sndb.TextProperty(
default='') # up to 10,000 characters
# "Associated Measures"
associated_measures = sndb.TextProperty(
default='') # Up to 5000 characters, HTML
# "Evidence of Effectiveness"
evidence_of_effectiveness = sndb.TextProperty(
default=''
) # Up to 10,000 characters, HTML text with ability to include links and images.
# "Related BELE Library Pages"
related_pages = sndb.StringProperty(
repeated=True) # up to 10 related pages.
# "Optional Contact Information"
acknowledgements = sndb.TextProperty(
default='') # Up to 5000 characters, HTML
# "Preferred Citation"
preferred_citation = sndb.StringProperty(
default='') # Up to 1000 characters, text
use_license = sndb.StringProperty(default='')
status = sndb.StringProperty(
default='pending',
choices=['draft', 'pending', 'approved', 'rejected',
'deleted']) # enum: draft, pending, approved, or rejected
####
mindset_tags = sndb.StringProperty(repeated=True)
page_tags = sndb.StringProperty(repeated=True)
time_of_year = sndb.StringProperty(default='')
class_period = sndb.StringProperty(default='')
type = sndb.StringProperty(default='text', search_type=search.AtomField)
body = sndb.TextProperty(default='', search_type=search.TextField)
youtube_id = sndb.StringProperty(default='')
iframe_src = sndb.StringProperty(default='')
has_files = sndb.BooleanProperty(default=False,
search_type=search.AtomField)
pending = sndb.BooleanProperty(default=True)
display_order = sndb.IntegerProperty(default=1,
search_type=search.NumberField)
votes_for = sndb.IntegerProperty(default=0, search_type=search.NumberField)
num_comments = sndb.IntegerProperty(default=0,
search_type=search.NumberField)
# class properties that contain files as json strings
file_props = ['icon']
@property
def ui_props(self):
return {
'time_required': {
'default_value': Page.time_required._default,
'data_prop': 'time_required',
'scope_prop': 'pageTime',
'heading_title': 'Time Required',
'value_suffix': None,
},
'required_materials': {
'default_value': Page.required_materials._default,
'data_prop': 'required_materials',
'scope_prop': 'pageMaterials',
'heading_title': 'Required Materials',
'value_suffix': None,
},
'preconditions_for_success': {
'default_value': Page.preconditions_for_success._default,
'data_prop': 'preconditions_for_success',
'scope_prop': 'pagePreconditions',
'heading_title': 'Preconditions for Success',
'value_suffix': None,
},
'advances_equity': {
'default_value': Page.advances_equity._default,
'data_prop': 'advances_equity',
'scope_prop': 'advancesEquity',
'heading_title': 'Connection to Equity',
'value_suffix': None,
},
'evidence_of_effectiveness': {
'default_value': Page.evidence_of_effectiveness._default,
'data_prop': 'evidence_of_effectiveness',
'scope_prop': 'pageEvidence',
'heading_title': 'Evidence of Effectiveness',
'value_suffix': None,
},
'associated_measures': {
'default_value': Page.associated_measures._default,
'data_prop': 'associated_measures',
'scope_prop': 'pageMeasures',
'heading_title': 'Associated Measures',
'value_suffix': None,
},
'acknowledgements': {
'default_value': Page.acknowledgements._default,
'data_prop': 'acknowledgements',
'scope_prop': 'pageAcknowledgements',
'heading_title': 'Acknowledgements',
'value_suffix': None,
},
'preferred_citation': {
'default_value': Page.preferred_citation._default,
'data_prop': 'preferred_citation',
'scope_prop': 'pageCitations',
'heading_title': 'Preferred Citation',
'value_suffix': None,
},
}
@classmethod
def create(klass, **kwargs):
"""Sends email to interested parties.
"""
page = super(klass, klass).create(**kwargs)
# Email interested parties that a page has been uploaded.
# mandrill.send(
# to_address=config.page_upload_recipients,
# subject="Page Uploaded to Mindset Kit!",
# template="page_upload_notification.html",
# template_data={'user': page.get_parent_user(),
# 'page': page,
# 'domain': 'https://{}'.format(os.environ['HOSTING_DOMAIN'])},
# )
# logging.info('model.Page queueing an email to: {}'
# .format(config.page_upload_recipients))
logging.info('creating page')
return page
@classmethod
def convert_uid(klass, short_or_long_uid):
"""Changes long-form uid's to short ones, and vice versa.
Overrides method provided in Model.
Long form example: Page_Pb4g9gus
Short form exmaple: Pb4g9gus
"""
if '_' in short_or_long_uid:
return ''.join([short_or_long_uid.split('_')[1]])
else:
return 'Page_{}'.format(short_or_long_uid[:4])
@classmethod
def get_long_uid(klass, short_or_long_uid):
"""Changes short of long-form uid's to long ones.
Overrides method provided in Model.
Long form example: Page_Pb4g9gus.User_oha4tp8a
Short form exmaple: Pb4g9gusoha4tp8a
"""
if '_' in short_or_long_uid: # is long
return short_or_long_uid
else: # is short
return 'Page_{}'.format(short_or_long_uid[-8:])
@classmethod
def get_popular_pages(klass):
"""Fetches popular pages to display on landing page
@todo: figure out a way to generate this list
- Possibly adding a field to pages and flagging x pages / week
- Needs more discussion
"""
pages = []
query = Page.query(
Page.deleted == False,
Page.listed == True,
Page.promoted == True,
)
query.order(-Page.created)
pages = query.fetch(20)
if len(pages) > 6:
pages = random.sample(pages, 6)
return pages
def add_file_data(self, file_dicts, entity_field=None):
"""Save dictionaries of uploaded file meta data."""
# Process specific field json files, specified by entity_field parameter
entity_fields_whitelist = self.file_props
if entity_field in entity_fields_whitelist:
for k, v in file_dicts[0].items():
if hasattr(v, 'isoformat'):
file_dicts[0][k] = v.isoformat()
setattr(self, entity_field,
json.dumps(file_dicts[0], default=util.json_dumps_default))
jp = self.json_properties
# Process generic json files if not an entity field
if entity_field is None:
if 'files' not in jp:
jp['files'] = []
jp['files'].extend(file_dicts)
self.json_properties = jp
self.has_files = 'files' in jp and len(jp['files']) > 0
def remove_file_data(self, file_key):
"""Remove file dictionaries from existing json_properties"""
jp = self.json_properties
# Find and remove file from 'files' in json_properties
if 'files' in jp:
for index, file_dict in enumerate(jp['files']):
if file_key == file_dict[u'gs_object_name']:
jp['files'].pop(index)
self.json_properties = jp
self.has_files = len(jp['files']) > 0
def get_parent_user(self):
return {}
def check_status_update(self, **kwargs):
"""Checks the status of an updated page to determine if the creator
should be notified of approval or rejection.
"""
new_status = kwargs.get('status', None)
now_reviewed = new_status in ('approved', 'rejected')
changing = self.status != new_status
if not now_reviewed or not changing:
return
authors = User.get_by_id(self.authors) or []
for author in authors:
short_name = author.first_name or ''
full_name = author.full_name
if new_status == 'approved':
# Send acceptance message
# @todo: add name to subject line
mandrill.send(
to_address=author.email,
subject="Your page upload is approved!",
template="accepted_notification.html",
template_data={
'short_name': short_name,
'full_name': full_name,
'entity_name': self.title,
'entity_url': '/pages/' + self.short_uid,
'domain':
'https://{}'.format(os.environ['HOSTING_DOMAIN']),
'year': datetime.date.today().year,
},
)
else:
# Send rejection message
mandrill.send(
to_address=author.email,
subject="We couldn't approve your page...",
template="rejected_notification.html",
template_data={
'short_name': short_name,
'full_name': full_name,
'entity_name': self.title,
'entity_url': '/pages/' + self.short_uid,
'edit_url': '/pages/edit/' + self.short_uid,
'domain':
'https://{}'.format(os.environ['HOSTING_DOMAIN']),
'year': datetime.date.today().year,
},
)
def to_search_document(self, rank=None):
"""Extends inherited method in Model."""
fields = super(Page, self)._get_search_fields()
# Add information about the parent user to the search document.
for author_id in self.authors:
author = self.get_by_id(author_id)
# Allow for empty first/last names, and default to an empty string.
if author is not None:
user_name = ''.join([(author.first_name or ''),
(author.last_name or '')])
fields.append(search.TextField(name='author', value=user_name))
# Simplify checking for video and file attachments
if self.has_files:
fields.append(search.AtomField(name='content_type', value='files'))
if self.youtube_id != '':
fields.append(search.AtomField(name='content_type', value='video'))
return search.Document(doc_id=self.uid,
fields=fields,
rank=rank,
language='en')
def to_client_dict(self):
d = super(Page, self).to_client_dict()
official_tags = []
additional_tags = []
for tag in d['tags']:
if tag in OFFICIAL_TAGS:
official_tags.append(tag)
else:
additional_tags.append(tag)
d['tags'] = official_tags
d['additional_tags'] = ', '.join(additional_tags)
return d
class User(Model):
"""Serve as root entities for Comments and Practices."""
# see config.auth_types
last_login = sndb.DateTimeProperty(auto_now_add=True)
auth_id = sndb.StringProperty(required=True,
validator=lambda prop, value: value.lower())
facebook_id = sndb.StringProperty(default=None,
search_type=search.TextField)
google_id = sndb.StringProperty(default=None, search_type=search.TextField)
hashed_password = sndb.StringProperty(default=None)
first_name = sndb.StringProperty(default='', search_type=search.TextField)
last_name = sndb.StringProperty(default='', search_type=search.TextField)
email = sndb.StringProperty(required=True,
validator=lambda prop, value: value.lower(),
search_type=search.TextField)
is_admin = sndb.BooleanProperty(default=False)
receives_updates = sndb.BooleanProperty(default=True)
image_url = sndb.StringProperty(default='')
short_bio = sndb.StringProperty(default='')
# Username for display
username = sndb.StringProperty(search_type=search.TextField)
# Username for uniqueness checking
canonical_username = sndb.ComputedProperty(
lambda self: self.username.lower() if self.username else '')
authorship_invites = sndb.StringProperty(repeated=True, default=None)
# App Engine can only run pure-python external libraries, and so we can't get
# a native (C-based) implementation of bcrypt. Pure python implementations are
# so slow that [the feasible number of rounds is insecure][1]. This uses the
# [algorithm recommended by passlib][2].
# [1]: http://stackoverflow.com/questions/7027196/how-can-i-use-bcrypt-scrypt-on-appengine-for-python
# [2]: https://pythonhosted.org/passlib/new_app_quickstart.html#sha512-crypt
password_hashing_context = CryptContext(
schemes=['sha512_crypt', 'sha256_crypt'],
default='sha512_crypt',
all__vary_rounds=0.1,
# Can change hashing rounds here. 656,000 is the default.
# sha512_crypt__default_rounds=656000,
# sha256_crypt__default_rounds=656000,
)
@classmethod
def create(klass, check_uniqueness=True, **kwargs):
"""Checks email uniqueness before creating; raises DuplicateUser.
Checking uniqueness always generates a Unique entity (an entity of
class 'Unique') that permanently reserves the email address. But
checking is also optional, and can be passed as False for testing.
See http://webapp-improved.appspot.com/_modules/webapp2_extras/appengine/auth/models.html#User.create_user
"""
# If no username, generate one!
if 'username' not in kwargs:
kwargs['username'] = User.create_username(**kwargs)
else:
if User.is_valid_username(kwargs['username']):
raise InvalidUsername(
"Invalid username {}. Use only letters, numbers, dashes, and underscores."
.format(kwargs['username']))
# Check for uniqueness of email and username
if check_uniqueness:
uniqueness_key = 'User.email:' + kwargs['email']
is_unique_email = Unique.create(uniqueness_key)
if not is_unique_email:
raise DuplicateUser(
"There is already a user with email {}.".format(
kwargs['email']))
uniqueness_key = 'User.username:'******'username'].lower()
is_unique_username = Unique.create(uniqueness_key)
if not is_unique_username:
# Need to also delete the unused email!
# To ensure that this works on retry
is_unique_email.delete()
raise DuplicateUser(
"There is already a user with username {}.".format(
kwargs['username']))
# Register user for the mailchimp list
should_subscribe = kwargs.pop('should_subscribe', False)
if should_subscribe is not False:
fields = {}
if 'first_name' in kwargs:
fields['first_name'] = kwargs['first_name']
if 'last_name' in kwargs:
fields['last_name'] = kwargs['last_name']
subscribed = mailchimp.subscribe(kwargs['email'], **fields)
if subscribed:
kwargs['receives_updates'] = True
# Add new user to user full text search index
kwargs['listed'] = True
kwargs['created'] = datetime.datetime.now()
kwargs['modified'] = datetime.datetime.now()
newUser = super(klass, klass).create(**kwargs)
index = search.Index(config.user_index)
index.put(newUser.to_search_document())
return newUser
@classmethod
def hash_password(klass, password):
if re.match(config.password_pattern, password) is None:
raise BadPassword(u'Bad password: {}'.format(password))
return klass.password_hashing_context.encrypt(password)
@classmethod
def verify_password(klass, password, hashed_password):
return (klass.password_hashing_context.verify(
password, hashed_password) if hashed_password else False)
@classmethod
def get_auth_id(klass, auth_type, third_party_id):
if auth_type not in config.auth_types:
raise Exception("Invalid auth type: {}.".format(auth_type))
return '{}:{}'.format(auth_type, third_party_id)
@property
def auth_type(self):
return self.auth_id.split(':')[0]
@property
def full_name(self):
if self.first_name and self.last_name:
return self.first_name + ' ' + self.last_name
if self.first_name:
return self.first_name
return self.username
@property
def profile_image(self):
return self.image_url or '{}/static/images/default-user.png'.format(
os.environ['HOSTING_DOMAIN'])
def set_password(self, new_password):
"""May raise BadPassword."""
logging.info("Setting new password {} for user {}.".format(
new_password, self))
self.hashed_password = self.hash_password(new_password)
# Alert the user that their password has been changed.
mandrill.send(
to_address=self.email,
subject="Your BELE Library password has been changed.",
template="change_password.html",
template_data={
'domain': 'https://{}'.format(os.environ['HOSTING_DOMAIN']),
'year': datetime.date.today().year,
},
)
@classmethod
def check_email_uniqueness(self, email):
"""Check if email is used"""
unique = Unique.get_by_id('User.email:' + email.lower())
return unique is None
@classmethod
def check_username_uniqueness(self, username):
"""Check if username is used"""
unique = Unique.get_by_id('User.username:'******'[^A-Za-z0-9\-\_]')
return username != regex.sub('', username)
@classmethod
def create_username(klass, **kwargs):
unformatted_username = ''
if 'first_name' in kwargs and 'last_name' in kwargs:
if kwargs['last_name'] is not None:
unformatted_username = kwargs['first_name'] + kwargs[
'last_name']
else:
unformatted_username = kwargs['first_name']
elif 'email' not in kwargs:
logging.error(
u"User doesn't have an email address {}".format(kwargs))
else:
unformatted_username = kwargs['email'].split('@')[0]
regex = re.compile('[^A-Za-z0-9\-\_]')
username = regex.sub('', unformatted_username)
if not User.check_username_uniqueness(username):
# If not unique, iterate through available names
raw_username = username
for n in range(1, 1000):
username = raw_username + str(n)
if User.check_username_uniqueness(username):
break
if n == 999:
raise Exception(
"No unique username found after 1000 tries.")
return username
@classmethod
def set_subscription(self, email, should_subscribe):
"""Sets user's subscription status to Update List on mailchimp
"""
# Use 'resubscribe' and 'unsubscribe' methods
if should_subscribe is True:
subscribed = mailchimp.resubscribe(email)
# if resubscription failed, they might not be subscribed at all!
if not subscribed:
subscribed = mailchimp.subscribe(email)
elif should_subscribe is False:
subscribed = mailchimp.unsubscribe(email)
return subscribed
def _post_put_hook(self, value):
super(User, self)._post_put_hook(value)
index = search.Index(config.user_index)
if self.listed:
index.put(self.to_search_document())
else:
index.delete(self.uid)
def _post_delete_hook(self, key, value):
search.Index(config.user_index).delete(key.id())
super(User, self)._post_delete_hook(key, value)
def update_email(self, email):
# Remove old email from unique keys so it can be re-used!
old_email = self.email
uniqueness_key = 'User.email:' + old_email
Unique.delete_multi([uniqueness_key])
# Update to new email and check uniqueness
setattr(self, 'email', email)
uniqueness_key = 'User.email:' + email
unique = Unique.create(uniqueness_key)
if not unique:
raise DuplicateField(
"There is already a user with email {}.".format(email))
# Also need to update the user in our mailchimp
mailchimp.unsubscribe(old_email)
if self.receives_updates:
subscribed = mailchimp.subscribe(email)
# if subscription failed, they might need to resubscribe!
if not subscribed:
subscribed = mailchimp.resubscribe(email)
return unique
def update_username(self, username):
if self.username is not None:
uniqueness_key = 'User.username:'******'username', username)
uniqueness_key = 'User.username:'******'User.email:' + self.email
uniqueness_key_username = '******' + self.username
Unique.delete_multi([uniqueness_key_email, uniqueness_key_username])
def add_user_image(self, blob_key):
"""Save dictionary of user image urls.
Create permenant link to user image --
You can resize and crop the image dynamically by specifying
the arguments in the URL '=sxx' where xx is an integer from 0-1600
representing the length, in pixels, of the image's longest side
Reference: https://cloud.google.com/appengine/docs/python/images/#Python_Transforming_images_from_the_Blobstore
"""
image_url = images.get_serving_url(blob_key, secure_url=True)
self.image_url = image_url
# Create a UserImage object pointing to blobstore object
# Reference used for removal of old UserImages
user_image = UserImage.create(
user_id=self.uid,
blob_key=blob_key,
)
user_image.put()
def remove_user_image(self):
"""Removes existing user's image completely"""
self.image_url = ''
# Find associated UserImage
user_image = UserImage.query(user_id=self.uid)
# Remove permenant link for image
images.delete_serving_url(user_image.blob_key)
user_image.deleted = True
user_image.put()
class Email(Model):
"""An email in a queue (not necessarily one that has been sent).
Emails have regular fields, to, from, subject, body. Also a send date and
an sent boolean to support queuing.
Uses jinja to attempt to attempt to interpolate values in the
template_data dictionary into the body and subject.
Body text comes in two flavors: raw text and html. The raw text is
interpreted as markdown (after jinja processing) to auto-generate the html
version (templates/email.html is also used for default styles). Which
version users see depends on their email client.
Consequently, all emails should be composed in markdown.
"""
to_address = sndb.StringProperty(required=True)
from_address = sndb.StringProperty(required=True)
reply_to = sndb.StringProperty(default=config.from_server_email_address)
subject = sndb.StringProperty(default="A message from the Mindset Kit")
body = sndb.TextProperty()
html = sndb.TextProperty()
scheduled_date = sndb.DateProperty()
was_sent = sndb.BooleanProperty(default=False)
was_attempted = sndb.BooleanProperty(default=False)
errors = sndb.TextProperty()
@classmethod
def create(klass, template_data={}, **kwargs):
def render(s):
return jinja2.Environment().from_string(s).render(**template_data)
kwargs['subject'] = render(kwargs['subject'])
kwargs['body'] = render(kwargs['body'])
return super(klass, klass).create(**kwargs)
@classmethod
def we_are_spamming(self, email):
"""Did we recently send email to this recipient?"""
to = email.to_address
# We can spam admins, like
# so we white list them in the config file
if to in config.addresses_we_can_spam:
return False
# We can also spam admins living at a @mindsetkit.org
if to.endswith('mindsetkit.org'):
return False
# Temporary spamming override...
return False
since = datetime.datetime.utcnow() - datetime.timedelta(
days=config.suggested_delay_between_emails)
query = Email.query(Email.was_sent == True,
Email.scheduled_date >= since)
return query.count(limit=1) > 0
@classmethod
def send(self, email):
if self.we_are_spamming(email):
logging.error("We are spamming {}:\n{}"
.format(email.to_address, email.to_dict()))
# Note that we are attempting to send so that we don't keep attempting.
email.was_attempted = True
email.put()
# Debugging info
logging.info(u"sending email: {}".format(email.to_dict()))
logging.info(u"to: {}".format(email.to_address))
logging.info(u"subject: {}".format(email.subject))
logging.info(u"body:\n{}".format(email.body))
# Make html version if it has not been explicitly passed in.
if not email.html:
email.html = (
jinja2.Environment(loader=jinja2.FileSystemLoader('templates'))
.get_template('email.html')
.render({'email_body': markdown.markdown(email.body)})
)
# Try to email through App Engine's API.
mail.send_mail(email.from_address,
email.to_address,
email.subject,
email.body,
reply_to=email.reply_to,
html=email.html)
email.was_sent = True
logging.info("""Sent successfully!""")
email.put()
@classmethod
def fetch_next_pending_email(self):
to_send = Email.query(
Email.deleted == False,
Email.scheduled_date <= datetime.datetime.utcnow(),
Email.was_sent == False,
Email.was_attempted == False,
)
return to_send.get()
@classmethod
def send_pending_email(self):
"""Send the next unsent email in the queue.
We only send one email at a time; this allows us to raise errors for
each email and avoid sending some crazy huge mass mail.
"""
email = self.fetch_next_pending_email()
if email:
self.send(email)
return email
else:
return None