class Content(Model):
"""Ancestor class for curated content objects: Theme, Topic, Lesson, and
Practice.
"""
name = sndb.StringProperty(required=True, search_type=search.TextField)
summary = sndb.TextProperty(default='', search_type=search.TextField)
tags = sndb.StringProperty(repeated=True, search_type=search.AtomField)
subjects = sndb.StringProperty(repeated=True, search_type=search.AtomField)
min_grade = sndb.IntegerProperty(default=0, search_type=search.NumberField)
max_grade = sndb.IntegerProperty(default=13,
search_type=search.NumberField)
promoted = sndb.BooleanProperty(default=False,
search_type=search.AtomField)
class Assessment(Model):
"""A MSK representation of a mindset meter assessment, created by PERTS.
Mostly just a name to list avaiable assessments and to organize surveys
created from them. The bulk of the definition of an assessment happens on
[mindsetmeter](survey.perts.net) in the form of html templates for surveys
and reports.
Always available to everyone, so listed by default.
"""
name = sndb.StringProperty(required=True, search_type=search.TextField)
# Defined on the client, but typically: self.name.lower().replace(' ', '-')
# So for 'Growth Mindset' this is 'growth-mindset', and it's used to make
# URLs for mindset meter, so users would be directed to
# 'survey.perts.net/take/growth-mindset'
url_name = sndb.StringProperty(required=True, validator=url_name_validator)
description = sndb.TextProperty(default='', search_type=search.TextField)
num_phases = sndb.IntegerProperty(required=True)
listed = sndb.BooleanProperty(default=True)
class Email(Model):
"""An email in a queue (not necessarily one that has been sent).
Emails have regular fields, to, from, subject, body. Also a send date and
an sent boolean to support queuing.
Uses jinja to attempt to attempt to interpolate values in the
template_data dictionary into the body and subject.
Body text comes in two flavors: raw text and html. The raw text is
interpreted as markdown (after jinja processing) to auto-generate the html
version (templates/email.html is also used for default styles). Which
version users see depends on their email client.
Consequently, all emails should be composed in markdown.
"""
to_address = sndb.StringProperty(required=True)
from_address = sndb.StringProperty(
default=config.from_server_email_address)
reply_to = sndb.StringProperty(default=config.from_server_email_address)
subject = sndb.StringProperty(default="A message from the Mindset Kit")
body = sndb.TextProperty()
html = sndb.TextProperty()
template = sndb.StringProperty()
template_data_string = sndb.TextProperty(default='{}')
scheduled_date = sndb.DateProperty(auto_now_add=True)
was_sent = sndb.BooleanProperty(default=False)
was_attempted = sndb.BooleanProperty(default=False)
errors = sndb.TextProperty()
@property
def template_data(self):
return json.loads(self.template_data_string)
@template_data.setter
def template_data(self, value):
self.template_data_string = json.dumps(value,
default=util.json_dumps_default)
return value
@classmethod
def create(klass, **kwargs):
"""
WARNING: Do NOT pass any objects in kwargs['template_data']
And be sure any associated templates use plain values in jinja code
"""
email = super(klass, klass).create(**kwargs)
return email
@classmethod
def we_are_spamming(self, email):
"""Did we recently send email to this recipient?
@todo discuss - I can think of cases where this can be harmful...
IE 2 different users comment on 2 different uploads from same user
I guess we should introduce Notifications in the app :)
"""
to = email.to_address
# We can spam admins, like
# so we white list them in the config file
if to in config.addresses_we_can_spam:
return False
# We can also spam admins living at a @mindsetkit.org
if to.endswith('mindsetkit.org'):
return False
since = datetime.datetime.utcnow() - datetime.timedelta(
minutes=config.suggested_delay_between_emails)
query = Email.query(
Email.was_sent == True,
Email.scheduled_date >= since,
Email.to_address == to,
)
return query.count(limit=1) > 0
@classmethod
def send(self, emails):
to_addresses = []
for email in emails:
if self.we_are_spamming(email):
# Do not send
# This user has already recieved very recent emails
# Debugging info
logging.error("We are spamming {}:\n{}".format(
email.to_address, email.to_dict()))
elif email.to_address in to_addresses:
# Do not send
# We don't send multiple emails to an address per 'send'
# Debugging info
logging.error("We are spamming {}:\n{}".format(
email.to_address, email.to_dict()))
else:
# Not spam! Let's send it
to_addresses.append(email.to_address)
# Debugging info
logging.info(u"sending email: {}".format(email.to_dict()))
logging.info(u"to: {}".format(email.to_address))
logging.info(u"subject: {}".format(email.subject))
if email.body:
logging.info(u"body:\n{}".format(email.body))
mandrill.send(
to_address=email.to_address,
subject=email.subject,
body=email.body,
template_data=email.template_data,
)
elif email.template:
logging.info(u"template: {}".format(email.template))
mandrill.send(
to_address=email.to_address,
subject=email.subject,
template=email.template,
template_data=email.template_data,
)
email.was_sent = True
logging.info("""Sent successfully!""")
# Note that we are attempting to send so that we don't keep attempting.
email.was_attempted = True
email.put()
@classmethod
def fetch_pending_emails(self):
to_send = Email.query(
Email.deleted == False,
Email.scheduled_date <= datetime.datetime.utcnow(),
Email.was_sent == False,
Email.was_attempted == False,
)
return to_send.fetch()
@classmethod
def send_pending_email(self):
"""Send the next unsent emails in the queue.
"""
emails = self.fetch_pending_emails()
if emails:
self.send(emails)
return emails
else:
return None
class Model(ndb.Model):
"""Superclass for all others; contains generic properties and methods."""
# This id is an encoding of the entity's key. For root entities (those
# with no ancestors), it's a class name (same as a GAE "kind") and
# an identifier separated by an underscore.
# Example: Theme_mW4iQ4cO
# For entities in a group (those with ancestors), their full heirarchy is
# encoded, ending with the root entity, separated by periods.
# Example: Comment_p46aOHS6.User_80h41Q4c
# An id is always sufficient to look up an entity directly.
# N.B.: We can't just call it 'id' because that breaks ndb.
uid = sndb.TextProperty(default=None)
short_uid = sndb.TextProperty(default=None, search_type=search.AtomField)
deleted = sndb.BooleanProperty(default=False)
created = sndb.DateTimeProperty(auto_now_add=True,
search_type=search.DateField)
modified = sndb.DateTimeProperty(auto_now=True,
search_type=search.DateField)
listed = sndb.BooleanProperty(default=False)
# Do NOT use sndb.JsonProperty here! That stores the data as a blob, which
# is DROPPED upon export to BigQuery. Instead, this stores the JSON as a
# string, which you can cleanly asscess as a python object through the
# `json_properties` property, below.
json_properties_string = sndb.TextProperty(default='{}')
@property
def json_properties(self):
return json.loads(self.json_properties_string)
@json_properties.setter
def json_properties(self, value):
self.json_properties_string = json.dumps(
value, default=util.json_dumps_default)
return value
@classmethod
def create(klass, **kwargs):
# ndb expects parents to be set with a key, but we find it more
# convenient to pass in entities. Do the translation here.
if 'parent' in kwargs:
parent = kwargs['parent'] # used in id generation
kwargs['parent'] = kwargs['parent'].key
else:
parent = None
if 'id' in kwargs:
# User has supplied their own id. That's okay (it makes certain
# URLs nice and readable), but it's not a real uid yet b/c it
# doesn't adhere to our ClassName_identifierXYZ convention. We'll
# pass it into generated_uid() later.
identifier = kwargs['id']
del kwargs['id']
else:
identifier = None
# ndb won't recognize the simulated json_properties if passed directly
# as a keyword argument. Translate that too.
if 'json_properties' in kwargs:
kwargs['json_properties_string'] = json.dumps(
kwargs['json_properties'])
del kwargs['json_properties']
# ndb won't recognize the simulated template_data if passed directly
# as a keyword argument. Translate that too.
if 'template_data' in kwargs:
kwargs['template_data_string'] = json.dumps(
kwargs['template_data'])
del kwargs['template_data']
# Make sure id is unique, otherwise "creating" this entity will
# overwrite an existing one, which could be a VERY hard bug to chase
# down.
for x in range(5):
uid = klass.generate_uid(parent, identifier)
existing_entity = Model.get_by_id(uid)
if not existing_entity:
break
if existing_entity:
if identifier:
raise Exception("Entity {} already exists.".format(uid))
else:
raise Exception("After five tries, could not generate a "
"unique id. This should NEVER happen.")
kwargs['uid'] = uid
kwargs['short_uid'] = klass.convert_uid(uid)
return klass(id=uid, **kwargs)
@classmethod
def generate_uid(klass, parent=None, identifier=None, existing=False):
"""Make a gobally unique id string, e.g. 'Program_mW4iQ4cO'.
Using 8 random chars, if we made 10,000 entities of the same kind (and
comments by a single user count as a kind), the probability of
duplication is 2E-7. Combined with the five attempts at uniqueness
in Model.create(), chances of duplication are essentially nil.
http://en.wikipedia.org/wiki/Universally_unique_identifier#Random_UUID_probability_of_duplicates
If a parent entity is specified, it becomes part of the uid, like this:
Comment_p46aOHS6.User_80h41Q4c
If an identifier is specified, it is used instead of random characters:
Theme_growth-mindset-is-cool
"""
if identifier:
if not re.match(r'^[A-Za-z0-9\-]+$', identifier):
raise Exception("Invalid identifier: {}. Letters, numbers, "
"and hyphens only.".format(identifier))
suffix = identifier
else:
chars = (string.ascii_uppercase + string.ascii_lowercase +
string.digits)
suffix = ''.join(random.choice(chars) for x in range(8))
uid = klass.__name__ + '_' + suffix
# Because comments exist as descendants of other entities, a simple
# id-as-key-name is insufficient. We must store information about its
# ancestry as well. Example:
# Comment_p46aOHS6.User_80h41Q4c
if parent:
uid += '.' + parent.uid
return uid
@classmethod
def convert_uid(klass, short_or_long_uid):
"""Changes long-form uid's to short ones, and vice versa.
Long form example: Theme_growth-mindset-is-cool
Short form exmaple: growth-mindset-is-cool
"""
if '_' in short_or_long_uid:
return short_or_long_uid.split('_')[1]
else:
return klass.generate_uid(identifier=short_or_long_uid)
@classmethod
def get_long_uid(klass, short_or_long_uid):
"""Changes short or long-form uid's to long ones.
Long form example: Theme_growth-mindset-is-cool
Short form exmaple: growth-mindset-is-cool
"""
if '_' in short_or_long_uid:
return short_or_long_uid
else:
# Check the characters to make sure this could be valid
if re.match(r'^[A-Za-z0-9\-]+$', short_or_long_uid):
return klass.generate_uid(identifier=short_or_long_uid)
else:
# Original UID is invalid anyway, return None
None
@classmethod
def get_parent_uid(klass, uid):
"""Don't use the datastore; get parent ids based on convention."""
if '.' not in uid:
raise Exception("Can't get parent of id: {}".format(uid))
return '.'.join(uid.split('.')[1:])
@classmethod
def get_kind(klass, obj):
"""Get the kind (class name string) of an entity, key, or id.
Examples:
* For a Theme entity, the kind is 'Theme'
* For the id 'Comment_p46aOHS6.User_80h41Q4c',
the kind is 'Comment'.
"""
if isinstance(obj, basestring):
return str(obj.split('_')[0])
elif isinstance(obj, Model):
return obj.__class__.__name__
elif isinstance(obj, ndb.Key):
return obj.kind()
else:
raise Exception('Model.get_kind() invalid input: {} ({}).'.format(
obj, str(type(obj))))
@classmethod
def get_class(klass, kind):
"""Convert a class name string (same as a GAE kind) to a class.
See http://stackoverflow.com/questions/1176136/convert-string-to-python-class-object
"""
return getattr(sys.modules['model'], kind, None)
@classmethod
def id_to_key(klass, id):
parts = id.split('.')
pairs = [(Model.get_kind(p), '.'.join(parts[-x:]))
for x, p in enumerate(parts)]
return ndb.Key(pairs=reversed(pairs))
@classmethod
def get_by_id(klass, id_or_list):
"""The main way to get entities with known ids.
Args:
id_or_list: A single perts id string, or a list of such strings,
of any kind or mixed kinds.
Returns an entity or list of entities, depending on input.
"""
# Sanitize input to a list of strings.
if type(id_or_list) in [str, unicode]:
ids = [id_or_list]
elif type(id_or_list) is list:
ids = id_or_list
else:
# I don't think we should be blocking code here
# Problem was occuring when you search a bad id or just None
# Ex. "/topics/foobar."
return None
raise Exception("Invalid id / id: {}.".format(id_or_list))
# Iterate through the list, generating a key for each id
keys = []
for id in ids:
keys.append(Model.id_to_key(id))
results = ndb.get_multi(keys)
# Wrangle results into expected structure.
if len(results) is 0:
return None
if type(id_or_list) in [str, unicode]:
return results[0]
if type(id_or_list) is list:
return results
def __str__(self):
"""A string represenation of the entity. Goal is to be readable.
Returns, e.g. <id_model.User User_oha4tp8a>.
Native implementation does nothing useful.
"""
return '<{}>'.format(self.key.id())
def __repr__(self):
"""A unique representation of the entity. Goal is to be unambiguous.
But our ids are unambiguous, so we can just forward to __str__.
Native implemention returns a useless memory address, e.g.
<id_model.User 0xa5e418cdd>
The big benefit here is to be able to print debuggable lists of
entities, without need to manipulate them first, e.g.
print [entity.id for entity in entity_list]
Now you can just write
print entity_list
"""
return self.__str__()
# special methods to allow comparing entities, even if they're different
# instances according to python
# https://groups.google.com/forum/?fromgroups=#!topic/google-appengine-python/uYneYzhIEzY
def __eq__(self, value):
"""Allows entity == entity to be True if keys match.
Is NOT called by `foo is bar`."""
if self.__class__ == value.__class__:
return self.key.id() == value.key.id()
else:
return False
# Because we defined the 'equals' method, eq, we must also be sure to
# define the 'not equals' method, ne, otherwise you might get A == B is
# True, and A != B is also True!
def __ne__(self, value):
"""Allows entity != entity to be False if keys match."""
if self.__class__ == value.__class__:
# if they're the same class, compare their keys
return self.key.id() != value.key.id()
else:
# if they're not the same class, then it's definitely True that
# they're not equal.
return True
def __hash__(self):
"""Allows entity in entity_list to be True."""
return hash(str(self.key))
def _post_put_hook(self, future):
"""Executes after an entity is put.
1. Updates search index
To allow for batch processing (i.e. doing the stuff this function does
for many entities all at once, instead of doing it here one by one),
this function can be disabled by adding an emphemeral attribute (one
not saved to the datastore) to the entity. Example:
# Don't do post-put processing for these entities, so we can handle
# them in a batch later.
for e in entities:
e.forbid_post_put_hook = True
ndb.put_multi(entities)
"""
if getattr(self, 'forbid_post_put_hook', False):
return
if self.get_kind(self) in config.indexed_models:
index = search.Index(config.content_index)
if self.listed:
logging.info("Indexing content for search.")
index.put(self.to_search_document())
else:
# Unlisted entities should be actively removed from search.
logging.info("Removing unlisted content from search.")
index.delete(self.uid)
@classmethod
def _post_delete_hook(klass, key, future):
"""We rarely truely delete entities, but when we do, we prefer Dos
Equis. I mean, we want to delete them from the search index."""
if klass.get_kind(key) in config.indexed_models:
logging.info(
"Removing hard-deleted content from search: {}".format(
key.id()))
search.Index(config.content_index).delete(key.id())
def to_client_dict(self, override=None):
"""Convert an app engine entity to a dictionary.
Ndb provides a to_dict() method, but we want to add creature-features:
1. Put properties in a predictable order so they're easier to read.
2. Remove or mask certain properties based on the preferences of our
javascript.
3. Handle our string-based json_properties correctly.
4. Ndb (different from db) stores datetimes as true python datetimes,
which JSON.dumps() doesn't know how to handle. We'll convert them
to ISO strings (e.g. "2010-04-20T20:08:21.634121")
Args:
override: obj, if provided, method turns this object into
a dictionary, rather than self.
"""
output = self.to_dict()
output['json_properties'] = self.json_properties
del output['json_properties_string']
for k, v in output.items():
if isinstance(v, dict):
for k2, v2 in output.items():
if hasattr(v2, 'isoformat'):
output[k][k2] = v2.isoformat()
if hasattr(v, 'isoformat'):
output[k] = v.isoformat()
client_safe_output = {}
for k, v in output.items():
if k in config.client_private_properties:
client_safe_output['_' + k] = v
elif k not in config.client_hidden_properties:
client_safe_output[k] = v
# order them so they're easier to read
ordered_dict = collections.OrderedDict(
sorted(client_safe_output.items(), key=lambda t: t[0]))
return ordered_dict
def _get_search_fields(self):
fields = []
klass = self.__class__
# Scan the properties in entity's class definition for information on
# how the entity should be converted to a search document. Properties
# can have search field types defined; we use those to construct the
# document.
for prop_name in dir(klass):
# This is the abstract property object in the class definition.
# E.g. sndb.StringProperty()
prop = getattr(klass, prop_name)
# This is the actual data defined on the entity
value = getattr(self, prop_name)
# This is (maybe) the type of search field the property should be
# converted to.
search_type = getattr(prop, 'search_type', None)
# The dir() function iterates all object attributes; only deal with
# those which 1) are datastore properties, 2) aren't private, and
# 3) have a search type defined.
is_field = (isinstance(prop, ndb.model.Property)
and not prop_name.startswith('_') and search_type)
if is_field:
# Search documents field names aren't unique; storing a list of
# values means making many fields of the same name. For
# brevity, make everything as a (possibly single-element) list.
if not type(value) is list:
value = [value]
for v in value:
# It will probably be common to put a boolean in a search
# document. The easiest way is to make it a string in an
# atom field.
if type(v) is bool and search_type is search.AtomField:
v = 'true' if v else 'false'
fields.append(search_type(name=prop_name, value=v))
# kind is not a field, but it's useful as a filter.
fields.append(search.AtomField(name='kind',
value=Model.get_kind(self)))
return fields
def to_search_document(self, rank=None):
fields = self._get_search_fields()
# Our uid's are already globally unique, so it's fine (as well as
# extremely convenient) to use them as search document ids as well.
# N.B. english language hard coded here. If we ever internationalize,
# will need to be dynamic.
return search.Document(doc_id=self.uid,
fields=fields,
rank=rank,
language='en')
class Comment(Model):
"""Always in a group under a User."""
body = sndb.TextProperty(default='')
practice_id = sndb.StringProperty(default=None)
lesson_id = sndb.StringProperty(default=None)
page_id = sndb.StringProperty(default=None)
# Overrides Model's default for this property, which is False. We always
# want to see comments.
listed = sndb.BooleanProperty(default=True)
@classmethod
def create(klass, **kwargs):
if ('practice_id' not in kwargs) and ('lesson_id' not in kwargs) and (
'page_id' not in kwargs):
raise Exception(
'Must specify a practice, lesson, or page when '
'creating a comment. Received kwargs: {}'.format(kwargs))
comment = super(klass, klass).create(**kwargs)
# For email notifications
content_url = '/'
content = None
if comment.page_id:
page = Page.get_by_id(comment.page_id)
if page is not None:
page.num_comments += 1
page.put()
# For email
content = page
content_url = '/pages/{}'.format(page.short_uid)
# Send email to creator
author_ids = page.authors
commenter = comment.get_parent_user()
authors = User.get_by_id(author_ids) or []
for author in authors:
# logic to not email yourself...
if author != commenter.email:
short_name = author.first_name or ''
full_name = author.full_name
commenter_image_url = commenter.profile_image
# Uses Email model to queue email and prevent spam
email = Email.create(
to_address=author.email,
subject=
"Someone commented on your BELE Library upload",
template="comment_creator_notification.html",
template_data={
'short_name':
short_name,
'full_name':
full_name,
'commenter_name':
commenter.full_name,
'commenter_image_url':
commenter_image_url,
'content_name':
content.title,
'comment_body':
comment.body,
'content_url':
content_url,
'domain':
'https://{}'.format(
os.environ['HOSTING_DOMAIN']),
'year':
datetime.date.today().year,
},
)
email.put()
# Send email to any users @replied to
usernames = re.search('\@(\w+)', comment.body)
if usernames is not None:
username = usernames.group(0).split('@')[1]
# Fetch user from username and send email message
replied_to = User.query(
User.username == username).fetch(1)
if replied_to:
replied_to = replied_to[0]
short_name = replied_to.first_name or ''
full_name = replied_to.full_name
commenter_image_url = commenter.profile_image
# Uses Email model to queue email and prevent spam
email = Email.create(
to_address=replied_to.email,
subject=
"Someone replied to you on BELE Library",
template="comment_reply_notification.html",
template_data={
'short_name':
short_name,
'full_name':
full_name,
'commenter_name':
commenter.full_name,
'commenter_image_url':
commenter_image_url,
'content_name':
content.title,
'comment_body':
comment.body,
'content_url':
content_url,
'domain':
'https://{}'.format(
os.environ['HOSTING_DOMAIN']),
'year':
datetime.date.today().year,
},
)
email.put()
# Email interested team members that a comment has been created
mandrill.send(
to_address=config.comment_recipients,
subject="New Comment on BELE Library!",
template="comment_notification.html",
template_data={
'comment': comment,
'user': comment.get_parent_user(),
'content_name': content.title,
'content_url': content_url,
'domain': 'https://{}'.format(os.environ['HOSTING_DOMAIN'])
},
)
logging.info('model.Comment queueing an email to: {}'.format(
config.comment_recipients))
return comment
@classmethod
def convert_uid(klass, short_or_long_uid):
"""Changes long-form uid's to short ones, and vice versa.
Overrides method provided in Model.
Long form example: Practice_Pb4g9gus.User_oha4tp8a
Short form exmaple: Pb4g9gusoha4tp8a
"""
if '.' in short_or_long_uid:
parts = short_or_long_uid.split('.')
return ''.join([x.split('_')[1] for x in parts])
else:
return 'Comment_{}.User_{}'.format(short_or_long_uid[:8],
short_or_long_uid[8:])
def parent_user_id(self):
return self.key.parent().id()
def get_parent_user(self):
return self.key.parent().get()
class Practice(Content):
"""Always in a group under a User."""
mindset_tags = sndb.StringProperty(repeated=True)
practice_tags = sndb.StringProperty(repeated=True)
time_of_year = sndb.StringProperty(default='')
class_period = sndb.StringProperty(default='')
# Stores the UID of 'associated' content (Theme or Topic)
# Used to fetch related practices from various pages
associated_content = sndb.StringProperty(default='')
type = sndb.StringProperty(default='text', search_type=search.AtomField)
body = sndb.TextProperty(default='', search_type=search.TextField)
youtube_id = sndb.StringProperty(default='')
iframe_src = sndb.StringProperty(default='')
has_files = sndb.BooleanProperty(default=False,
search_type=search.AtomField)
pending = sndb.BooleanProperty(default=True)
votes_for = sndb.IntegerProperty(default=0, search_type=search.NumberField)
num_comments = sndb.IntegerProperty(default=0,
search_type=search.NumberField)
@classmethod
def create(klass, **kwargs):
"""Sends email to interested parties.
"""
practice = super(klass, klass).create(**kwargs)
# Email interested parties that a practice has been uploaded.
mandrill.send(
to_address=config.practice_upload_recipients,
subject="Practice Uploaded to Mindset Kit!",
template="practice_upload_notification.html",
template_data={
'user': practice.get_parent_user(),
'practice': practice,
'domain': os.environ['HOSTING_DOMAIN']
},
)
logging.info('model.Practice queueing an email to: {}'.format(
config.practice_upload_recipients))
return practice
@classmethod
def convert_uid(klass, short_or_long_uid):
"""Changes long-form uid's to short ones, and vice versa.
Overrides method provided in Model.
Long form example: Practice_Pb4g9gus.User_oha4tp8a
Short form exmaple: Pb4g9gusoha4tp8a
"""
if '.' in short_or_long_uid:
parts = short_or_long_uid.split('.')
return ''.join([x.split('_')[1] for x in parts])
else:
return 'Practice_{}.User_{}'.format(short_or_long_uid[:8],
short_or_long_uid[8:])
@classmethod
def get_long_uid(klass, short_or_long_uid):
"""Changes short of long-form uid's to long ones.
Overrides method provided in Model.
Long form example: Practice_Pb4g9gus.User_oha4tp8a
Short form exmaple: Pb4g9gusoha4tp8a
"""
if '.' in short_or_long_uid: # is long
return short_or_long_uid
else: # is short
return 'Practice_{}.User_{}'.format(short_or_long_uid[:8],
short_or_long_uid[8:])
@classmethod
def get_related_practices(klass, content, count):
"""Fetches practices related to a content object
Will default to random pratices if none found.
"""
related_practices = []
query = Practice.query(
Practice.deleted == False,
Practice.listed == True,
)
if content.uid:
if Model.get_kind(content.uid) == 'Practice':
if content.associated_content:
query = query.filter(Practice.associated_content ==
content.associated_content)
query = query.filter(Practice.uid != content.uid)
else:
query = query.filter(
Practice.associated_content == content.uid)
query.order(-Practice.created)
# Pull a 'bucket' of practices to sample from
related_practices_bucket = query.fetch(15)
# Only return a random selection of the practices
if len(related_practices_bucket) > count:
related_practices = random.sample(related_practices_bucket, count)
elif len(related_practices_bucket) <= 0:
related_practices = []
else:
related_practices = related_practices_bucket
return related_practices
@classmethod
def get_popular_practices(klass):
"""Fetches popular practices to display on landing page
@todo: figure out a way to generate this list
- Possibly adding a field to practices and flagging x practices / week
- Needs more discussion
"""
practices = []
query = Practice.query(
Practice.deleted == False,
Practice.listed == True,
Practice.promoted == True,
)
query.order(-Practice.created)
practices = query.fetch(20)
if len(practices) > 6:
practices = random.sample(practices, 6)
return practices
def add_file_data(self, file_dicts):
"""Save dictionaries of uploaded file meta data."""
jp = self.json_properties
if 'files' not in jp:
jp['files'] = []
jp['files'].extend(file_dicts)
self.json_properties = jp
self.has_files = len(jp['files']) > 0
def remove_file_data(self, file_key):
"""Remove file dictionaries from existing json_properties"""
jp = self.json_properties
# Find and remove file from 'files' in json_properties
if 'files' in jp:
for index, file_dict in enumerate(jp['files']):
if file_key == file_dict[u'gs_object_name']:
jp['files'].pop(index)
self.json_properties = jp
self.has_files = len(jp['files']) > 0
def get_parent_user(self):
return self.key.parent().get()
def check_status_update(self, **kwargs):
"""Checks the status of an updated practice to determine if the creator
should be notified of approval or rejection
Only triggered if pending set from True to False (prevents duplicates)
"""
if (self.pending is True and kwargs.get('pending') is False):
creator = self.get_parent_user()
short_name = creator.first_name if creator.first_name else ''
full_name = creator.full_name
if (self.listed is False and kwargs.get('listed') is True):
# Send acceptance message
# @todo: add name to subject line
mandrill.send(
to_address=creator.email,
subject="Your practice upload is approved!",
template="accepted_notification.html",
template_data={
'short_name': short_name,
'full_name': full_name,
'practice_name': self.name,
'practice_url': '/practices/' + self.short_uid,
'domain': os.environ['HOSTING_DOMAIN']
},
)
else:
# Send rejection message
mandrill.send(
to_address=creator.email,
subject="We couldn't approve your practice...",
template="rejected_notification.html",
template_data={
'short_name': short_name,
'full_name': full_name,
'practice_name': self.name,
'practice_url': '/practices/' + self.short_uid,
'edit_url': '/practices/edit/' + self.short_uid,
'domain': os.environ['HOSTING_DOMAIN']
},
)
def to_search_document(self, rank=None):
"""Extends inherited method in Model."""
fields = super(Practice, self)._get_search_fields()
# Add information about the parent user to the search document.
user = self.get_parent_user()
# Allow for empty first/last names, and default to an empty string.
if user is not None:
user_name = ''.join([(user.first_name or ''), (user.last_name
or '')])
fields.append(search.TextField(name='author', value=user_name))
# Simplify checking for video and file attachments
if self.has_files:
fields.append(search.AtomField(name='content_type', value='files'))
if self.youtube_id != '':
fields.append(search.AtomField(name='content_type', value='video'))
return search.Document(doc_id=self.uid,
fields=fields,
rank=rank,
language='en')
class Page(Content):
####
chapters = sndb.StringProperty(repeated=True) # unordered parents
authors = sndb.StringProperty(
repeated=True, search_type=search.TextField) # unordered children
icon = sndb.StringProperty(default='', search_type=search.TextField)
iconPath = sndb.ComputedProperty(
lambda self: util.extract_value_from_json(self.icon, 'link'))
# "Challenges & Preconditions"
preconditions_for_success = sndb.TextProperty(
default=''
) # Up to 10,000 characters, HTML text with ability to include links and images.
# "Connection to Equity"
advances_equity = sndb.TextProperty(
default='') # up to 5,000 characters, HTML
# "Time Required"
time_required = sndb.StringProperty(default='') # minutes
# "Required Materials"
required_materials = sndb.TextProperty(
default='') # up to 10,000 characters
# "Associated Measures"
associated_measures = sndb.TextProperty(
default='') # Up to 5000 characters, HTML
# "Evidence of Effectiveness"
evidence_of_effectiveness = sndb.TextProperty(
default=''
) # Up to 10,000 characters, HTML text with ability to include links and images.
# "Related BELE Library Pages"
related_pages = sndb.StringProperty(
repeated=True) # up to 10 related pages.
# "Optional Contact Information"
acknowledgements = sndb.TextProperty(
default='') # Up to 5000 characters, HTML
# "Preferred Citation"
preferred_citation = sndb.StringProperty(
default='') # Up to 1000 characters, text
use_license = sndb.StringProperty(default='')
status = sndb.StringProperty(
default='pending',
choices=['draft', 'pending', 'approved', 'rejected',
'deleted']) # enum: draft, pending, approved, or rejected
####
mindset_tags = sndb.StringProperty(repeated=True)
page_tags = sndb.StringProperty(repeated=True)
time_of_year = sndb.StringProperty(default='')
class_period = sndb.StringProperty(default='')
type = sndb.StringProperty(default='text', search_type=search.AtomField)
body = sndb.TextProperty(default='', search_type=search.TextField)
youtube_id = sndb.StringProperty(default='')
iframe_src = sndb.StringProperty(default='')
has_files = sndb.BooleanProperty(default=False,
search_type=search.AtomField)
pending = sndb.BooleanProperty(default=True)
display_order = sndb.IntegerProperty(default=1,
search_type=search.NumberField)
votes_for = sndb.IntegerProperty(default=0, search_type=search.NumberField)
num_comments = sndb.IntegerProperty(default=0,
search_type=search.NumberField)
# class properties that contain files as json strings
file_props = ['icon']
@property
def ui_props(self):
return {
'time_required': {
'default_value': Page.time_required._default,
'data_prop': 'time_required',
'scope_prop': 'pageTime',
'heading_title': 'Time Required',
'value_suffix': None,
},
'required_materials': {
'default_value': Page.required_materials._default,
'data_prop': 'required_materials',
'scope_prop': 'pageMaterials',
'heading_title': 'Required Materials',
'value_suffix': None,
},
'preconditions_for_success': {
'default_value': Page.preconditions_for_success._default,
'data_prop': 'preconditions_for_success',
'scope_prop': 'pagePreconditions',
'heading_title': 'Preconditions for Success',
'value_suffix': None,
},
'advances_equity': {
'default_value': Page.advances_equity._default,
'data_prop': 'advances_equity',
'scope_prop': 'advancesEquity',
'heading_title': 'Connection to Equity',
'value_suffix': None,
},
'evidence_of_effectiveness': {
'default_value': Page.evidence_of_effectiveness._default,
'data_prop': 'evidence_of_effectiveness',
'scope_prop': 'pageEvidence',
'heading_title': 'Evidence of Effectiveness',
'value_suffix': None,
},
'associated_measures': {
'default_value': Page.associated_measures._default,
'data_prop': 'associated_measures',
'scope_prop': 'pageMeasures',
'heading_title': 'Associated Measures',
'value_suffix': None,
},
'acknowledgements': {
'default_value': Page.acknowledgements._default,
'data_prop': 'acknowledgements',
'scope_prop': 'pageAcknowledgements',
'heading_title': 'Acknowledgements',
'value_suffix': None,
},
'preferred_citation': {
'default_value': Page.preferred_citation._default,
'data_prop': 'preferred_citation',
'scope_prop': 'pageCitations',
'heading_title': 'Preferred Citation',
'value_suffix': None,
},
}
@classmethod
def create(klass, **kwargs):
"""Sends email to interested parties.
"""
page = super(klass, klass).create(**kwargs)
# Email interested parties that a page has been uploaded.
# mandrill.send(
# to_address=config.page_upload_recipients,
# subject="Page Uploaded to Mindset Kit!",
# template="page_upload_notification.html",
# template_data={'user': page.get_parent_user(),
# 'page': page,
# 'domain': 'https://{}'.format(os.environ['HOSTING_DOMAIN'])},
# )
# logging.info('model.Page queueing an email to: {}'
# .format(config.page_upload_recipients))
logging.info('creating page')
return page
@classmethod
def convert_uid(klass, short_or_long_uid):
"""Changes long-form uid's to short ones, and vice versa.
Overrides method provided in Model.
Long form example: Page_Pb4g9gus
Short form exmaple: Pb4g9gus
"""
if '_' in short_or_long_uid:
return ''.join([short_or_long_uid.split('_')[1]])
else:
return 'Page_{}'.format(short_or_long_uid[:4])
@classmethod
def get_long_uid(klass, short_or_long_uid):
"""Changes short of long-form uid's to long ones.
Overrides method provided in Model.
Long form example: Page_Pb4g9gus.User_oha4tp8a
Short form exmaple: Pb4g9gusoha4tp8a
"""
if '_' in short_or_long_uid: # is long
return short_or_long_uid
else: # is short
return 'Page_{}'.format(short_or_long_uid[-8:])
@classmethod
def get_popular_pages(klass):
"""Fetches popular pages to display on landing page
@todo: figure out a way to generate this list
- Possibly adding a field to pages and flagging x pages / week
- Needs more discussion
"""
pages = []
query = Page.query(
Page.deleted == False,
Page.listed == True,
Page.promoted == True,
)
query.order(-Page.created)
pages = query.fetch(20)
if len(pages) > 6:
pages = random.sample(pages, 6)
return pages
def add_file_data(self, file_dicts, entity_field=None):
"""Save dictionaries of uploaded file meta data."""
# Process specific field json files, specified by entity_field parameter
entity_fields_whitelist = self.file_props
if entity_field in entity_fields_whitelist:
for k, v in file_dicts[0].items():
if hasattr(v, 'isoformat'):
file_dicts[0][k] = v.isoformat()
setattr(self, entity_field,
json.dumps(file_dicts[0], default=util.json_dumps_default))
jp = self.json_properties
# Process generic json files if not an entity field
if entity_field is None:
if 'files' not in jp:
jp['files'] = []
jp['files'].extend(file_dicts)
self.json_properties = jp
self.has_files = 'files' in jp and len(jp['files']) > 0
def remove_file_data(self, file_key):
"""Remove file dictionaries from existing json_properties"""
jp = self.json_properties
# Find and remove file from 'files' in json_properties
if 'files' in jp:
for index, file_dict in enumerate(jp['files']):
if file_key == file_dict[u'gs_object_name']:
jp['files'].pop(index)
self.json_properties = jp
self.has_files = len(jp['files']) > 0
def get_parent_user(self):
return {}
def check_status_update(self, **kwargs):
"""Checks the status of an updated page to determine if the creator
should be notified of approval or rejection.
"""
new_status = kwargs.get('status', None)
now_reviewed = new_status in ('approved', 'rejected')
changing = self.status != new_status
if not now_reviewed or not changing:
return
authors = User.get_by_id(self.authors) or []
for author in authors:
short_name = author.first_name or ''
full_name = author.full_name
if new_status == 'approved':
# Send acceptance message
# @todo: add name to subject line
mandrill.send(
to_address=author.email,
subject="Your page upload is approved!",
template="accepted_notification.html",
template_data={
'short_name': short_name,
'full_name': full_name,
'entity_name': self.title,
'entity_url': '/pages/' + self.short_uid,
'domain':
'https://{}'.format(os.environ['HOSTING_DOMAIN']),
'year': datetime.date.today().year,
},
)
else:
# Send rejection message
mandrill.send(
to_address=author.email,
subject="We couldn't approve your page...",
template="rejected_notification.html",
template_data={
'short_name': short_name,
'full_name': full_name,
'entity_name': self.title,
'entity_url': '/pages/' + self.short_uid,
'edit_url': '/pages/edit/' + self.short_uid,
'domain':
'https://{}'.format(os.environ['HOSTING_DOMAIN']),
'year': datetime.date.today().year,
},
)
def to_search_document(self, rank=None):
"""Extends inherited method in Model."""
fields = super(Page, self)._get_search_fields()
# Add information about the parent user to the search document.
for author_id in self.authors:
author = self.get_by_id(author_id)
# Allow for empty first/last names, and default to an empty string.
if author is not None:
user_name = ''.join([(author.first_name or ''),
(author.last_name or '')])
fields.append(search.TextField(name='author', value=user_name))
# Simplify checking for video and file attachments
if self.has_files:
fields.append(search.AtomField(name='content_type', value='files'))
if self.youtube_id != '':
fields.append(search.AtomField(name='content_type', value='video'))
return search.Document(doc_id=self.uid,
fields=fields,
rank=rank,
language='en')
def to_client_dict(self):
d = super(Page, self).to_client_dict()
official_tags = []
additional_tags = []
for tag in d['tags']:
if tag in OFFICIAL_TAGS:
official_tags.append(tag)
else:
additional_tags.append(tag)
d['tags'] = official_tags
d['additional_tags'] = ', '.join(additional_tags)
return d
class Email(Model):
"""An email in a queue (not necessarily one that has been sent).
Emails have regular fields, to, from, subject, body. Also a send date and
an sent boolean to support queuing.
Uses jinja to attempt to attempt to interpolate values in the
template_data dictionary into the body and subject.
Body text comes in two flavors: raw text and html. The raw text is
interpreted as markdown (after jinja processing) to auto-generate the html
version (templates/email.html is also used for default styles). Which
version users see depends on their email client.
Consequently, all emails should be composed in markdown.
"""
to_address = sndb.StringProperty(required=True)
from_address = sndb.StringProperty(required=True)
reply_to = sndb.StringProperty(default=config.from_server_email_address)
subject = sndb.StringProperty(default="A message from the Mindset Kit")
body = sndb.TextProperty()
html = sndb.TextProperty()
scheduled_date = sndb.DateProperty()
was_sent = sndb.BooleanProperty(default=False)
was_attempted = sndb.BooleanProperty(default=False)
errors = sndb.TextProperty()
@classmethod
def create(klass, template_data={}, **kwargs):
def render(s):
return jinja2.Environment().from_string(s).render(**template_data)
kwargs['subject'] = render(kwargs['subject'])
kwargs['body'] = render(kwargs['body'])
return super(klass, klass).create(**kwargs)
@classmethod
def we_are_spamming(self, email):
"""Did we recently send email to this recipient?"""
to = email.to_address
# We can spam admins, like
# so we white list them in the config file
if to in config.addresses_we_can_spam:
return False
# We can also spam admins living at a @mindsetkit.org
if to.endswith('mindsetkit.org'):
return False
# Temporary spamming override...
return False
since = datetime.datetime.utcnow() - datetime.timedelta(
days=config.suggested_delay_between_emails)
query = Email.query(Email.was_sent == True,
Email.scheduled_date >= since)
return query.count(limit=1) > 0
@classmethod
def send(self, email):
if self.we_are_spamming(email):
logging.error("We are spamming {}:\n{}"
.format(email.to_address, email.to_dict()))
# Note that we are attempting to send so that we don't keep attempting.
email.was_attempted = True
email.put()
# Debugging info
logging.info(u"sending email: {}".format(email.to_dict()))
logging.info(u"to: {}".format(email.to_address))
logging.info(u"subject: {}".format(email.subject))
logging.info(u"body:\n{}".format(email.body))
# Make html version if it has not been explicitly passed in.
if not email.html:
email.html = (
jinja2.Environment(loader=jinja2.FileSystemLoader('templates'))
.get_template('email.html')
.render({'email_body': markdown.markdown(email.body)})
)
# Try to email through App Engine's API.
mail.send_mail(email.from_address,
email.to_address,
email.subject,
email.body,
reply_to=email.reply_to,
html=email.html)
email.was_sent = True
logging.info("""Sent successfully!""")
email.put()
@classmethod
def fetch_next_pending_email(self):
to_send = Email.query(
Email.deleted == False,
Email.scheduled_date <= datetime.datetime.utcnow(),
Email.was_sent == False,
Email.was_attempted == False,
)
return to_send.get()
@classmethod
def send_pending_email(self):
"""Send the next unsent email in the queue.
We only send one email at a time; this allows us to raise errors for
each email and avoid sending some crazy huge mass mail.
"""
email = self.fetch_next_pending_email()
if email:
self.send(email)
return email
else:
return None
class Book(Content):
chapters = sndb.StringProperty(repeated=True) # ordered children
authors = sndb.StringProperty(repeated=True, search_type=search.TextField) # unordered children
book_image = sndb.StringProperty(default='', search_type=search.TextField)
icon = sndb.StringProperty(default='', search_type=search.TextField)
status = sndb.StringProperty(default='pending', choices=['draft', 'pending', 'approved', 'rejected', 'deleted']) # enum: draft, pending, approved, or rejected
acknowledgements = sndb.TextProperty(default='') # up to 1000 chars, HTML
preferred_citation = sndb.StringProperty(default='') # up to 1000 chars, text
display_order = sndb.IntegerProperty(default=1,
search_type=search.NumberField)
votes_for = sndb.IntegerProperty(default=0, search_type=search.NumberField)
num_comments = sndb.IntegerProperty(default=0, search_type=search.NumberField)
locale = sndb.StringProperty(default='en')
# class properties that contain files as json strings
file_props = ['book_image', 'icon'];
def add_file_data(self, file_dicts, entity_field='files'):
"""Save dictionaries of uploaded file meta data."""
entity_fields_whitelist = self.file_props
if entity_field in entity_fields_whitelist:
setattr(self, entity_field, json.dumps(file_dicts[0], default=util.json_dumps_default))
def remove_file_data(self, file_key):
"""Remove file dictionaries from existing json_properties"""
for prop in self.file_props:
value = getattr(self, prop)
value = util.try_parse_json(value)
# delete file if matches old file, or if new value is empty string
if value == '' or file_key == value['gs_object_name']:
setattr(self, prop, '')
def check_status_update(self, **kwargs):
"""Checks the status of an updated page to determine if the creator
should be notified of approval or rejection.
"""
new_status = kwargs.get('status', None)
now_reviewed = new_status in ('approved', 'rejected')
changing = self.status != new_status
if not now_reviewed or not changing:
return
authors = User.get_by_id(self.authors) or []
for author in authors:
short_name = author.first_name or ''
full_name = author.full_name
if (new_status == 'approved'):
# Send acceptance message
# @todo: add name to subject line
mandrill.send(
to_address=author.email,
subject="Your book upload is approved!",
template="accepted_notification.html",
template_data={
'short_name': short_name,
'full_name': full_name,
'entity_name': self.title,
'entity_url': '/books/' + self.short_uid,
'domain': 'https://{}'.format(os.environ['HOSTING_DOMAIN']),
'year': datetime.date.today().year,
},
)
elif (new_status == 'rejected'):
# Send rejection message
mandrill.send(
to_address=author.email,
subject="We couldn't approve your book...",
template="rejected_notification.html",
template_data={
'short_name': short_name,
'full_name': full_name,
'entity_name': self.title,
'entity_url': '/books/' + self.short_uid,
'edit_url': '/books/manage/' + self.short_uid,
'domain': 'https://{}'.format(os.environ['HOSTING_DOMAIN']),
'year': datetime.date.today().year,
},
)
def get_chapters_with_pages(self, map_function=None):
"""Return ordered array of page data, organized by chapter"""
pages_dict = {};
chapters_dict = {};
chapters = Chapter.get_by_id(self.chapters)
if chapters is None:
return {}
chapter_pages = []
page_ids = []
for chapter in chapters:
if chapter:
chapters_dict[chapter.uid] = chapter.to_client_dict()
page_ids.extend(chapter.pages)
page_ids = list(set(page_ids))
pages = Page.get_by_id(page_ids)
pages = pages if pages is not None else []
for page in pages:
page.icon = util.extract_value_from_json(page.icon, 'link')
page.icon = page.icon + '?size=360' if page.icon else page.icon
pages_dict[page.uid] = page.to_client_dict()
# Returns a dict with the page uid and a link to the page detail
def get_page_dict(page_id):
page = pages_dict[page_id]
# page['icon'] = util.extract_value_from_json(page['icon'], 'link')
return page;
for c in chapters:
if c:
chapter_dict = chapters_dict[c.uid]
chapter_dict['pages'] = map(lambda page_id: get_page_dict(page_id), c.pages)
chapter_pages.append(chapter_dict)
return chapter_pages