class Model(ndb.Model):
"""Superclass for all others; contains generic properties and methods."""
# This id is an encoding of the entity's key. For root entities (those
# with no ancestors), it's a class name (same as a GAE "kind") and
# an identifier separated by an underscore.
# Example: Theme_mW4iQ4cO
# For entities in a group (those with ancestors), their full heirarchy is
# encoded, ending with the root entity, separated by periods.
# Example: Comment_p46aOHS6.User_80h41Q4c
# An id is always sufficient to look up an entity directly.
# N.B.: We can't just call it 'id' because that breaks ndb.
uid = sndb.TextProperty(default=None)
short_uid = sndb.TextProperty(default=None, search_type=search.AtomField)
deleted = sndb.BooleanProperty(default=False)
created = sndb.DateTimeProperty(auto_now_add=True,
search_type=search.DateField)
modified = sndb.DateTimeProperty(auto_now=True,
search_type=search.DateField)
listed = sndb.BooleanProperty(default=False)
# Do NOT use sndb.JsonProperty here! That stores the data as a blob, which
# is DROPPED upon export to BigQuery. Instead, this stores the JSON as a
# string, which you can cleanly asscess as a python object through the
# `json_properties` property, below.
json_properties_string = sndb.TextProperty(default='{}')
@property
def json_properties(self):
return json.loads(self.json_properties_string)
@json_properties.setter
def json_properties(self, value):
self.json_properties_string = json.dumps(
value, default=util.json_dumps_default)
return value
@classmethod
def create(klass, **kwargs):
# ndb expects parents to be set with a key, but we find it more
# convenient to pass in entities. Do the translation here.
if 'parent' in kwargs:
parent = kwargs['parent'] # used in id generation
kwargs['parent'] = kwargs['parent'].key
else:
parent = None
if 'id' in kwargs:
# User has supplied their own id. That's okay (it makes certain
# URLs nice and readable), but it's not a real uid yet b/c it
# doesn't adhere to our ClassName_identifierXYZ convention. We'll
# pass it into generated_uid() later.
identifier = kwargs['id']
del kwargs['id']
else:
identifier = None
# ndb won't recognize the simulated json_properties if passed directly
# as a keyword argument. Translate that too.
if 'json_properties' in kwargs:
kwargs['json_properties_string'] = json.dumps(
kwargs['json_properties'])
del kwargs['json_properties']
# ndb won't recognize the simulated template_data if passed directly
# as a keyword argument. Translate that too.
if 'template_data' in kwargs:
kwargs['template_data_string'] = json.dumps(
kwargs['template_data'])
del kwargs['template_data']
# Make sure id is unique, otherwise "creating" this entity will
# overwrite an existing one, which could be a VERY hard bug to chase
# down.
for x in range(5):
uid = klass.generate_uid(parent, identifier)
existing_entity = Model.get_by_id(uid)
if not existing_entity:
break
if existing_entity:
if identifier:
raise Exception("Entity {} already exists.".format(uid))
else:
raise Exception("After five tries, could not generate a "
"unique id. This should NEVER happen.")
kwargs['uid'] = uid
kwargs['short_uid'] = klass.convert_uid(uid)
return klass(id=uid, **kwargs)
@classmethod
def generate_uid(klass, parent=None, identifier=None, existing=False):
"""Make a gobally unique id string, e.g. 'Program_mW4iQ4cO'.
Using 8 random chars, if we made 10,000 entities of the same kind (and
comments by a single user count as a kind), the probability of
duplication is 2E-7. Combined with the five attempts at uniqueness
in Model.create(), chances of duplication are essentially nil.
http://en.wikipedia.org/wiki/Universally_unique_identifier#Random_UUID_probability_of_duplicates
If a parent entity is specified, it becomes part of the uid, like this:
Comment_p46aOHS6.User_80h41Q4c
If an identifier is specified, it is used instead of random characters:
Theme_growth-mindset-is-cool
"""
if identifier:
if not re.match(r'^[A-Za-z0-9\-]+$', identifier):
raise Exception("Invalid identifier: {}. Letters, numbers, "
"and hyphens only.".format(identifier))
suffix = identifier
else:
chars = (string.ascii_uppercase + string.ascii_lowercase +
string.digits)
suffix = ''.join(random.choice(chars) for x in range(8))
uid = klass.__name__ + '_' + suffix
# Because comments exist as descendants of other entities, a simple
# id-as-key-name is insufficient. We must store information about its
# ancestry as well. Example:
# Comment_p46aOHS6.User_80h41Q4c
if parent:
uid += '.' + parent.uid
return uid
@classmethod
def convert_uid(klass, short_or_long_uid):
"""Changes long-form uid's to short ones, and vice versa.
Long form example: Theme_growth-mindset-is-cool
Short form exmaple: growth-mindset-is-cool
"""
if '_' in short_or_long_uid:
return short_or_long_uid.split('_')[1]
else:
return klass.generate_uid(identifier=short_or_long_uid)
@classmethod
def get_long_uid(klass, short_or_long_uid):
"""Changes short or long-form uid's to long ones.
Long form example: Theme_growth-mindset-is-cool
Short form exmaple: growth-mindset-is-cool
"""
if '_' in short_or_long_uid:
return short_or_long_uid
else:
# Check the characters to make sure this could be valid
if re.match(r'^[A-Za-z0-9\-]+$', short_or_long_uid):
return klass.generate_uid(identifier=short_or_long_uid)
else:
# Original UID is invalid anyway, return None
None
@classmethod
def get_parent_uid(klass, uid):
"""Don't use the datastore; get parent ids based on convention."""
if '.' not in uid:
raise Exception("Can't get parent of id: {}".format(uid))
return '.'.join(uid.split('.')[1:])
@classmethod
def get_kind(klass, obj):
"""Get the kind (class name string) of an entity, key, or id.
Examples:
* For a Theme entity, the kind is 'Theme'
* For the id 'Comment_p46aOHS6.User_80h41Q4c',
the kind is 'Comment'.
"""
if isinstance(obj, basestring):
return str(obj.split('_')[0])
elif isinstance(obj, Model):
return obj.__class__.__name__
elif isinstance(obj, ndb.Key):
return obj.kind()
else:
raise Exception('Model.get_kind() invalid input: {} ({}).'.format(
obj, str(type(obj))))
@classmethod
def get_class(klass, kind):
"""Convert a class name string (same as a GAE kind) to a class.
See http://stackoverflow.com/questions/1176136/convert-string-to-python-class-object
"""
return getattr(sys.modules['model'], kind, None)
@classmethod
def id_to_key(klass, id):
parts = id.split('.')
pairs = [(Model.get_kind(p), '.'.join(parts[-x:]))
for x, p in enumerate(parts)]
return ndb.Key(pairs=reversed(pairs))
@classmethod
def get_by_id(klass, id_or_list):
"""The main way to get entities with known ids.
Args:
id_or_list: A single perts id string, or a list of such strings,
of any kind or mixed kinds.
Returns an entity or list of entities, depending on input.
"""
# Sanitize input to a list of strings.
if type(id_or_list) in [str, unicode]:
ids = [id_or_list]
elif type(id_or_list) is list:
ids = id_or_list
else:
# I don't think we should be blocking code here
# Problem was occuring when you search a bad id or just None
# Ex. "/topics/foobar."
return None
raise Exception("Invalid id / id: {}.".format(id_or_list))
# Iterate through the list, generating a key for each id
keys = []
for id in ids:
keys.append(Model.id_to_key(id))
results = ndb.get_multi(keys)
# Wrangle results into expected structure.
if len(results) is 0:
return None
if type(id_or_list) in [str, unicode]:
return results[0]
if type(id_or_list) is list:
return results
def __str__(self):
"""A string represenation of the entity. Goal is to be readable.
Returns, e.g. <id_model.User User_oha4tp8a>.
Native implementation does nothing useful.
"""
return '<{}>'.format(self.key.id())
def __repr__(self):
"""A unique representation of the entity. Goal is to be unambiguous.
But our ids are unambiguous, so we can just forward to __str__.
Native implemention returns a useless memory address, e.g.
<id_model.User 0xa5e418cdd>
The big benefit here is to be able to print debuggable lists of
entities, without need to manipulate them first, e.g.
print [entity.id for entity in entity_list]
Now you can just write
print entity_list
"""
return self.__str__()
# special methods to allow comparing entities, even if they're different
# instances according to python
# https://groups.google.com/forum/?fromgroups=#!topic/google-appengine-python/uYneYzhIEzY
def __eq__(self, value):
"""Allows entity == entity to be True if keys match.
Is NOT called by `foo is bar`."""
if self.__class__ == value.__class__:
return self.key.id() == value.key.id()
else:
return False
# Because we defined the 'equals' method, eq, we must also be sure to
# define the 'not equals' method, ne, otherwise you might get A == B is
# True, and A != B is also True!
def __ne__(self, value):
"""Allows entity != entity to be False if keys match."""
if self.__class__ == value.__class__:
# if they're the same class, compare their keys
return self.key.id() != value.key.id()
else:
# if they're not the same class, then it's definitely True that
# they're not equal.
return True
def __hash__(self):
"""Allows entity in entity_list to be True."""
return hash(str(self.key))
def _post_put_hook(self, future):
"""Executes after an entity is put.
1. Updates search index
To allow for batch processing (i.e. doing the stuff this function does
for many entities all at once, instead of doing it here one by one),
this function can be disabled by adding an emphemeral attribute (one
not saved to the datastore) to the entity. Example:
# Don't do post-put processing for these entities, so we can handle
# them in a batch later.
for e in entities:
e.forbid_post_put_hook = True
ndb.put_multi(entities)
"""
if getattr(self, 'forbid_post_put_hook', False):
return
if self.get_kind(self) in config.indexed_models:
index = search.Index(config.content_index)
if self.listed:
logging.info("Indexing content for search.")
index.put(self.to_search_document())
else:
# Unlisted entities should be actively removed from search.
logging.info("Removing unlisted content from search.")
index.delete(self.uid)
@classmethod
def _post_delete_hook(klass, key, future):
"""We rarely truely delete entities, but when we do, we prefer Dos
Equis. I mean, we want to delete them from the search index."""
if klass.get_kind(key) in config.indexed_models:
logging.info(
"Removing hard-deleted content from search: {}".format(
key.id()))
search.Index(config.content_index).delete(key.id())
def to_client_dict(self, override=None):
"""Convert an app engine entity to a dictionary.
Ndb provides a to_dict() method, but we want to add creature-features:
1. Put properties in a predictable order so they're easier to read.
2. Remove or mask certain properties based on the preferences of our
javascript.
3. Handle our string-based json_properties correctly.
4. Ndb (different from db) stores datetimes as true python datetimes,
which JSON.dumps() doesn't know how to handle. We'll convert them
to ISO strings (e.g. "2010-04-20T20:08:21.634121")
Args:
override: obj, if provided, method turns this object into
a dictionary, rather than self.
"""
output = self.to_dict()
output['json_properties'] = self.json_properties
del output['json_properties_string']
for k, v in output.items():
if isinstance(v, dict):
for k2, v2 in output.items():
if hasattr(v2, 'isoformat'):
output[k][k2] = v2.isoformat()
if hasattr(v, 'isoformat'):
output[k] = v.isoformat()
client_safe_output = {}
for k, v in output.items():
if k in config.client_private_properties:
client_safe_output['_' + k] = v
elif k not in config.client_hidden_properties:
client_safe_output[k] = v
# order them so they're easier to read
ordered_dict = collections.OrderedDict(
sorted(client_safe_output.items(), key=lambda t: t[0]))
return ordered_dict
def _get_search_fields(self):
fields = []
klass = self.__class__
# Scan the properties in entity's class definition for information on
# how the entity should be converted to a search document. Properties
# can have search field types defined; we use those to construct the
# document.
for prop_name in dir(klass):
# This is the abstract property object in the class definition.
# E.g. sndb.StringProperty()
prop = getattr(klass, prop_name)
# This is the actual data defined on the entity
value = getattr(self, prop_name)
# This is (maybe) the type of search field the property should be
# converted to.
search_type = getattr(prop, 'search_type', None)
# The dir() function iterates all object attributes; only deal with
# those which 1) are datastore properties, 2) aren't private, and
# 3) have a search type defined.
is_field = (isinstance(prop, ndb.model.Property)
and not prop_name.startswith('_') and search_type)
if is_field:
# Search documents field names aren't unique; storing a list of
# values means making many fields of the same name. For
# brevity, make everything as a (possibly single-element) list.
if not type(value) is list:
value = [value]
for v in value:
# It will probably be common to put a boolean in a search
# document. The easiest way is to make it a string in an
# atom field.
if type(v) is bool and search_type is search.AtomField:
v = 'true' if v else 'false'
fields.append(search_type(name=prop_name, value=v))
# kind is not a field, but it's useful as a filter.
fields.append(search.AtomField(name='kind',
value=Model.get_kind(self)))
return fields
def to_search_document(self, rank=None):
fields = self._get_search_fields()
# Our uid's are already globally unique, so it's fine (as well as
# extremely convenient) to use them as search document ids as well.
# N.B. english language hard coded here. If we ever internationalize,
# will need to be dynamic.
return search.Document(doc_id=self.uid,
fields=fields,
rank=rank,
language='en')
class User(Model):
"""Serve as root entities for Comments and Practices."""
# see config.auth_types
last_login = sndb.DateTimeProperty(auto_now_add=True)
auth_id = sndb.StringProperty(required=True,
validator=lambda prop, value: value.lower())
facebook_id = sndb.StringProperty(default=None,
search_type=search.TextField)
google_id = sndb.StringProperty(default=None, search_type=search.TextField)
hashed_password = sndb.StringProperty(default=None)
first_name = sndb.StringProperty(default='', search_type=search.TextField)
last_name = sndb.StringProperty(default='', search_type=search.TextField)
email = sndb.StringProperty(required=True,
validator=lambda prop, value: value.lower(),
search_type=search.TextField)
is_admin = sndb.BooleanProperty(default=False)
receives_updates = sndb.BooleanProperty(default=True)
image_url = sndb.StringProperty(default='')
short_bio = sndb.StringProperty(default='')
# Username for display
username = sndb.StringProperty(search_type=search.TextField)
# Username for uniqueness checking
canonical_username = sndb.ComputedProperty(
lambda self: self.username.lower() if self.username else '')
authorship_invites = sndb.StringProperty(repeated=True, default=None)
# App Engine can only run pure-python external libraries, and so we can't get
# a native (C-based) implementation of bcrypt. Pure python implementations are
# so slow that [the feasible number of rounds is insecure][1]. This uses the
# [algorithm recommended by passlib][2].
# [1]: http://stackoverflow.com/questions/7027196/how-can-i-use-bcrypt-scrypt-on-appengine-for-python
# [2]: https://pythonhosted.org/passlib/new_app_quickstart.html#sha512-crypt
password_hashing_context = CryptContext(
schemes=['sha512_crypt', 'sha256_crypt'],
default='sha512_crypt',
all__vary_rounds=0.1,
# Can change hashing rounds here. 656,000 is the default.
# sha512_crypt__default_rounds=656000,
# sha256_crypt__default_rounds=656000,
)
@classmethod
def create(klass, check_uniqueness=True, **kwargs):
"""Checks email uniqueness before creating; raises DuplicateUser.
Checking uniqueness always generates a Unique entity (an entity of
class 'Unique') that permanently reserves the email address. But
checking is also optional, and can be passed as False for testing.
See http://webapp-improved.appspot.com/_modules/webapp2_extras/appengine/auth/models.html#User.create_user
"""
# If no username, generate one!
if 'username' not in kwargs:
kwargs['username'] = User.create_username(**kwargs)
else:
if User.is_valid_username(kwargs['username']):
raise InvalidUsername(
"Invalid username {}. Use only letters, numbers, dashes, and underscores."
.format(kwargs['username']))
# Check for uniqueness of email and username
if check_uniqueness:
uniqueness_key = 'User.email:' + kwargs['email']
is_unique_email = Unique.create(uniqueness_key)
if not is_unique_email:
raise DuplicateUser(
"There is already a user with email {}.".format(
kwargs['email']))
uniqueness_key = 'User.username:'******'username'].lower()
is_unique_username = Unique.create(uniqueness_key)
if not is_unique_username:
# Need to also delete the unused email!
# To ensure that this works on retry
is_unique_email.delete()
raise DuplicateUser(
"There is already a user with username {}.".format(
kwargs['username']))
# Register user for the mailchimp list
should_subscribe = kwargs.pop('should_subscribe', False)
if should_subscribe is not False:
fields = {}
if 'first_name' in kwargs:
fields['first_name'] = kwargs['first_name']
if 'last_name' in kwargs:
fields['last_name'] = kwargs['last_name']
subscribed = mailchimp.subscribe(kwargs['email'], **fields)
if subscribed:
kwargs['receives_updates'] = True
# Add new user to user full text search index
kwargs['listed'] = True
kwargs['created'] = datetime.datetime.now()
kwargs['modified'] = datetime.datetime.now()
newUser = super(klass, klass).create(**kwargs)
index = search.Index(config.user_index)
index.put(newUser.to_search_document())
return newUser
@classmethod
def hash_password(klass, password):
if re.match(config.password_pattern, password) is None:
raise BadPassword(u'Bad password: {}'.format(password))
return klass.password_hashing_context.encrypt(password)
@classmethod
def verify_password(klass, password, hashed_password):
return (klass.password_hashing_context.verify(
password, hashed_password) if hashed_password else False)
@classmethod
def get_auth_id(klass, auth_type, third_party_id):
if auth_type not in config.auth_types:
raise Exception("Invalid auth type: {}.".format(auth_type))
return '{}:{}'.format(auth_type, third_party_id)
@property
def auth_type(self):
return self.auth_id.split(':')[0]
@property
def full_name(self):
if self.first_name and self.last_name:
return self.first_name + ' ' + self.last_name
if self.first_name:
return self.first_name
return self.username
@property
def profile_image(self):
return self.image_url or '{}/static/images/default-user.png'.format(
os.environ['HOSTING_DOMAIN'])
def set_password(self, new_password):
"""May raise BadPassword."""
logging.info("Setting new password {} for user {}.".format(
new_password, self))
self.hashed_password = self.hash_password(new_password)
# Alert the user that their password has been changed.
mandrill.send(
to_address=self.email,
subject="Your BELE Library password has been changed.",
template="change_password.html",
template_data={
'domain': 'https://{}'.format(os.environ['HOSTING_DOMAIN']),
'year': datetime.date.today().year,
},
)
@classmethod
def check_email_uniqueness(self, email):
"""Check if email is used"""
unique = Unique.get_by_id('User.email:' + email.lower())
return unique is None
@classmethod
def check_username_uniqueness(self, username):
"""Check if username is used"""
unique = Unique.get_by_id('User.username:'******'[^A-Za-z0-9\-\_]')
return username != regex.sub('', username)
@classmethod
def create_username(klass, **kwargs):
unformatted_username = ''
if 'first_name' in kwargs and 'last_name' in kwargs:
if kwargs['last_name'] is not None:
unformatted_username = kwargs['first_name'] + kwargs[
'last_name']
else:
unformatted_username = kwargs['first_name']
elif 'email' not in kwargs:
logging.error(
u"User doesn't have an email address {}".format(kwargs))
else:
unformatted_username = kwargs['email'].split('@')[0]
regex = re.compile('[^A-Za-z0-9\-\_]')
username = regex.sub('', unformatted_username)
if not User.check_username_uniqueness(username):
# If not unique, iterate through available names
raw_username = username
for n in range(1, 1000):
username = raw_username + str(n)
if User.check_username_uniqueness(username):
break
if n == 999:
raise Exception(
"No unique username found after 1000 tries.")
return username
@classmethod
def set_subscription(self, email, should_subscribe):
"""Sets user's subscription status to Update List on mailchimp
"""
# Use 'resubscribe' and 'unsubscribe' methods
if should_subscribe is True:
subscribed = mailchimp.resubscribe(email)
# if resubscription failed, they might not be subscribed at all!
if not subscribed:
subscribed = mailchimp.subscribe(email)
elif should_subscribe is False:
subscribed = mailchimp.unsubscribe(email)
return subscribed
def _post_put_hook(self, value):
super(User, self)._post_put_hook(value)
index = search.Index(config.user_index)
if self.listed:
index.put(self.to_search_document())
else:
index.delete(self.uid)
def _post_delete_hook(self, key, value):
search.Index(config.user_index).delete(key.id())
super(User, self)._post_delete_hook(key, value)
def update_email(self, email):
# Remove old email from unique keys so it can be re-used!
old_email = self.email
uniqueness_key = 'User.email:' + old_email
Unique.delete_multi([uniqueness_key])
# Update to new email and check uniqueness
setattr(self, 'email', email)
uniqueness_key = 'User.email:' + email
unique = Unique.create(uniqueness_key)
if not unique:
raise DuplicateField(
"There is already a user with email {}.".format(email))
# Also need to update the user in our mailchimp
mailchimp.unsubscribe(old_email)
if self.receives_updates:
subscribed = mailchimp.subscribe(email)
# if subscription failed, they might need to resubscribe!
if not subscribed:
subscribed = mailchimp.resubscribe(email)
return unique
def update_username(self, username):
if self.username is not None:
uniqueness_key = 'User.username:'******'username', username)
uniqueness_key = 'User.username:'******'User.email:' + self.email
uniqueness_key_username = '******' + self.username
Unique.delete_multi([uniqueness_key_email, uniqueness_key_username])
def add_user_image(self, blob_key):
"""Save dictionary of user image urls.
Create permenant link to user image --
You can resize and crop the image dynamically by specifying
the arguments in the URL '=sxx' where xx is an integer from 0-1600
representing the length, in pixels, of the image's longest side
Reference: https://cloud.google.com/appengine/docs/python/images/#Python_Transforming_images_from_the_Blobstore
"""
image_url = images.get_serving_url(blob_key, secure_url=True)
self.image_url = image_url
# Create a UserImage object pointing to blobstore object
# Reference used for removal of old UserImages
user_image = UserImage.create(
user_id=self.uid,
blob_key=blob_key,
)
user_image.put()
def remove_user_image(self):
"""Removes existing user's image completely"""
self.image_url = ''
# Find associated UserImage
user_image = UserImage.query(user_id=self.uid)
# Remove permenant link for image
images.delete_serving_url(user_image.blob_key)
user_image.deleted = True
user_image.put()
class User(Model):
"""Serve as root entities for Comments and Practices."""
# see config.auth_types
last_login = sndb.DateTimeProperty(auto_now_add=True)
auth_id = sndb.StringProperty(required=True,
validator=lambda prop, value: value.lower())
facebook_id = sndb.StringProperty(default=None)
google_id = sndb.StringProperty(default=None)
hashed_password = sndb.StringProperty(default=None)
first_name = sndb.StringProperty(default='')
last_name = sndb.StringProperty(default='')
email = sndb.StringProperty(required=True,
validator=lambda prop, value: value.lower())
is_admin = sndb.BooleanProperty(default=False)
receives_updates = sndb.BooleanProperty(default=True)
image_url = sndb.StringProperty(default='')
short_bio = sndb.StringProperty(default='')
# Username for display
username = sndb.StringProperty()
# Username for uniqueness checking
canonical_username = sndb.ComputedProperty(
lambda self: self.username.lower() if self.username else '')
@classmethod
def create(klass, check_uniqueness=True, **kwargs):
"""Checks email uniqueness before creating; raises DuplicateUser.
Checking uniqueness always generates a Unique entity (an entity of
class 'Unique') that permanently reserves the email address. But
checking is also optional, and can be passed as False for testing.
See http://webapp-improved.appspot.com/_modules/webapp2_extras/appengine/auth/models.html#User.create_user
"""
# If no username, generate one!
if 'username' not in kwargs:
kwargs['username'] = User.create_username(**kwargs)
else:
if User.is_valid_username(kwargs['username']):
raise InvalidUsername(
"Invalid username {}. Use only letters, numbers, dashes, and underscores."
.format(kwargs['username']))
# Check for uniqueness of email and username
if check_uniqueness:
uniqueness_key = 'User.email:' + kwargs['email']
is_unique_email = Unique.create(uniqueness_key)
if not is_unique_email:
raise DuplicateUser(
"There is already a user with email {}.".format(
kwargs['email']))
uniqueness_key = 'User.username:'******'username'].lower()
is_unique_username = Unique.create(uniqueness_key)
if not is_unique_username:
# Need to also delete the unused email!
# To ensure that this works on retry
is_unique_email.delete()
raise DuplicateUser(
"There is already a user with username {}.".format(
kwargs['username']))
# Register user for the mailchimp list
should_subscribe = kwargs.pop('should_subscribe', False)
if should_subscribe is not False:
fields = {}
if 'first_name' in kwargs:
fields['first_name'] = kwargs['first_name']
if 'last_name' in kwargs:
fields['last_name'] = kwargs['last_name']
subscribed = mailchimp.subscribe(kwargs['email'], **fields)
if subscribed:
kwargs['receives_updates'] = True
return super(klass, klass).create(**kwargs)
@classmethod
def get_auth_id(klass, auth_type, third_party_id):
if auth_type not in config.auth_types:
raise Exception("Invalid auth type: {}.".format(auth_type))
return '{}:{}'.format(auth_type, third_party_id)
@property
def auth_type(self):
return self.auth_id.split(':')[0]
@property
def full_name(self):
if self.first_name and self.last_name:
return self.first_name + ' ' + self.last_name
else:
return self.username
@property
def profile_image(self):
if self.image_url:
return self.image_url
elif self.facebook_id:
return 'http://graph.facebook.com/{}/picture?type=square'.format(
self.facebook_id)
else:
return 'https://www.mindsetkit.org/static/images/default-user.png'
def set_password(self, new_password):
"""May raise util.BadPassword."""
logging.info("Setting new password {} for user {}.".format(
new_password, self))
self.hashed_password = util.hash_password(new_password)
# Alert the user that their password has been changed.
mandrill.send(
to_address=self.email,
subject="Your Mindset Kit password has been changed.",
template="change_password.html",
)
logging.info('User.set_password queueing an email to: {}'.format(
self.email))
@classmethod
def check_email_uniqueness(self, email):
"""Check if email is used"""
unique = Unique.get_by_id('User.email:' + email.lower())
return unique is None
@classmethod
def check_username_uniqueness(self, username):
"""Check if username is used"""
unique = Unique.get_by_id('User.username:'******'[^A-Za-z0-9\-\_]')
return username != regex.sub('', username)
@classmethod
def create_username(klass, **kwargs):
unformatted_username = ''
if 'first_name' in kwargs and 'last_name' in kwargs:
if kwargs['last_name'] is not None:
unformatted_username = kwargs['first_name'] + kwargs[
'last_name']
else:
unformatted_username = kwargs['first_name']
elif 'email' not in kwargs:
logging.error(
u"User doesn't have an email address {}".format(kwargs))
else:
unformatted_username = kwargs['email'].split('@')[0]
regex = re.compile('[^A-Za-z0-9\-\_]')
username = regex.sub('', unformatted_username)
if not User.check_username_uniqueness(username):
# If not unique, iterate through available names
raw_username = username
for n in range(1, 1000):
username = raw_username + str(n)
if User.check_username_uniqueness(username):
break
if n == 999:
raise Exception(
"No unique username found after 1000 tries.")
return username
@classmethod
def set_subscription(self, email, should_subscribe):
"""Sets user's subscription status to Update List on mailchimp
"""
# Use 'resubscribe' and 'unsubscribe' methods
if should_subscribe is True:
subscribed = mailchimp.resubscribe(email)
# if resubscription failed, they might not be subscribed at all!
if not subscribed:
subscribed = mailchimp.subscribe(email)
elif should_subscribe is False:
subscribed = mailchimp.unsubscribe(email)
return subscribed
def update_email(self, email):
# Remove old email from unique keys so it can be re-used!
old_email = self.email
uniqueness_key = 'User.email:' + old_email
Unique.delete_multi([uniqueness_key])
# Update to new email and check uniqueness
setattr(self, 'email', email)
uniqueness_key = 'User.email:' + email
unique = Unique.create(uniqueness_key)
if not unique:
raise DuplicateField(
"There is already a user with email {}.".format(email))
# Also need to update the user in our mailchimp
mailchimp.unsubscribe(old_email)
if self.receives_updates:
subscribed = mailchimp.subscribe(email)
# if subscription failed, they might need to resubscribe!
if not subscribed:
subscribed = mailchimp.resubscribe(email)
return unique
def update_username(self, username):
if self.username is not None:
uniqueness_key = 'User.username:'******'username', username)
uniqueness_key = 'User.username:'******'User.email:' + self.email
uniqueness_key_username = '******' + self.username
Unique.delete_multi([uniqueness_key_email, uniqueness_key_username])
def add_user_image(self, blob_key):
"""Save dictionary of user image urls.
Create permenant link to user image --
You can resize and crop the image dynamically by specifying
the arguments in the URL '=sxx' where xx is an integer from 0-1600
representing the length, in pixels, of the image's longest side
Reference: https://cloud.google.com/appengine/docs/python/images/#Python_Transforming_images_from_the_Blobstore
"""
image_url = images.get_serving_url(blob_key)
self.image_url = image_url
# Create a UserImage object pointing to blobstore object
# Reference used for removal of old UserImages
user_image = UserImage.create(
user_id=self.uid,
blob_key=blob_key,
)
user_image.put()
def remove_user_image(self):
"""Removes existing user's image completely"""
self.image_url = ''
# Find associated UserImage
user_image = UserImage.query(user_id=self.uid)
# Remove permenant link for image
images.delete_serving_url(user_image.blob_key)
user_image.deleted = True
user_image.put()
class ErrorChecker(ndb.Model):
"""
Check for recent errors using log api
Design
The error checker will keep track of how long it has been since a check
occured and how long since an email alert was sent.
It will also facilite searching the error log.
orginal author
bmh October 2013
"""
# constants
# How long will we wait between emails?
minimum_seconds_between_emails = 60 * 60 # 1 hour
maximum_requests_to_email = 100 # how long can the log be
maximum_entries_per_request = 100 # how long can the log be
# error levels
level_map = collections.defaultdict(lambda x: 'UNKNOWN')
level_map[logservice.LOG_LEVEL_DEBUG] = 'DEBUG'
level_map[logservice.LOG_LEVEL_INFO] = 'INFO'
level_map[logservice.LOG_LEVEL_WARNING] = 'WARNING'
level_map[logservice.LOG_LEVEL_ERROR] = 'ERROR'
level_map[logservice.LOG_LEVEL_CRITICAL] = 'CRITICAL'
# email stuff
to_addresses = config.to_dev_team_email_addresses
from_address = config.from_server_email_address
subject = "Error(s) during calls to: "
body = ("General Krang,\n\n"
"The continuing growth of your brain is threatened. More "
"information is available on the dashboard.\n\n"
"https://console.developers.google.com/project/mindsetkit/logs\n\n"
"We haven't taken over the world, YET.\n\n"
"The Mindset Kit\n")
# Data
last_check = sndb.DateTimeProperty()
last_email = sndb.DateTimeProperty()
def datetime(self):
return datetime.datetime.utcnow()
def to_unix_time(self, dt):
return calendar.timegm(dt.timetuple())
def to_utc_time(self, unix_time):
return datetime.datetime.utcfromtimestamp(unix_time)
def any_new_errors(self):
since = self.last_check if self.last_check else self.datetime()
log_stream = logservice.fetch(
start_time=self.to_unix_time(since),
minimum_log_level=logservice.LOG_LEVEL_ERROR
)
return next(iter(log_stream), None) is not None
def get_recent_log(self):
""" see api
https://developers.google.com/appengine/docs/python/logs/functions
"""
out = ""
since = self.last_check if self.last_check else self.datetime()
log_stream = logservice.fetch(
start_time=self.to_unix_time(since),
minimum_log_level=logservice.LOG_LEVEL_ERROR,
include_app_logs=True
)
requests = itertools.islice(
log_stream, 0, self.maximum_requests_to_email)
for r in requests:
log = itertools.islice(
r.app_logs, 0, self.maximum_entries_per_request)
log = [
self.level_map[l.level] + '\t' +
str(self.to_utc_time(l.time)) + '\t' +
l.message + '\n'
for l in log
]
out = out + r.combined + '\n' + ''.join(log) + '\n\n'
return out
def get_error_summary(self):
""" A short high level overview of the error.
This was designed to serve as the email subject line so that
developers could quickly see if an error was a new type of error.
It returns the resources that were requested as a comma
seperated string:
e.g.
/api/put/pd, /api/...
see google api
https://developers.google.com/appengine/docs/python/logs/functions
"""
# Get a record of all the requests which generated an error
# since the last check was performed, typically this will be
# at most one error, but we don't want to ignore other errors if
# they occurred.
since = self.last_check if self.last_check else self.datetime()
log_stream = logservice.fetch(
start_time=self.to_unix_time(since),
minimum_log_level=logservice.LOG_LEVEL_ERROR,
include_app_logs=True
)
# Limit the maximum number of errors that will be processed
# to avoid insane behavior that should never happen, like
# emailing a report with a googleplex error messages.
requests = itertools.islice(
log_stream, 0, self.maximum_requests_to_email
)
# This should return a list of any requested resources
# that led to an error. Usually there will only be one.
# for example:
# /api/put/pd
# or
# /api/put/pd, /api/another_call
out = ', '.join(set([r.resource for r in requests]))
return out
def should_email(self):
since_last = ((self.datetime() - self.last_email).seconds
if self.last_email else 10000000)
return since_last > self.minimum_seconds_between_emails
def mail_log(self):
body = self.body + self.get_recent_log()
subject = self.subject + self.get_error_summary()
# Ignore the normal email queueing / spam-prevention system because the
# addressees are devs, and they can customize the deluge themselves.
for to in self.to_addresses:
# We want to send this immediately, not in batches.
mandrill.send(
to_address=to,
subject=subject,
body=body,
)
self.last_email = self.now
return (subject, body)
def check(self):
self.now = self.datetime()
should_email = self.should_email()
new_errors = self.any_new_errors()
# check for errors
if new_errors and should_email:
message = self.mail_log()
else:
message = None
logging.info("any_new_errors: {}, should_email: {}, message: {}"
.format(new_errors, should_email,
'None' if message is None else message[0]))
self.last_check = self.now
# TODO(benjaminhaley) this should return simpler output, ala
# chris's complaint https://github.com/daveponet/pegasus/pull/197/files#diff-281842ae8036e3fcb830df255cd15610R663
return {
'email content': message,
'checked for new errors': should_email
}