def init_context(self):
# public api
self.p_api = Api(User(user_type='public'))
# This closely follows the pattern of the populate script in god.html
self.researcher = User(user_type='researcher')
self.researcher.put()
self.r_api = Api(self.researcher)
self.teacher = User.create(user_type='teacher')
self.teacher.put()
self.t_api = Api(self.teacher)
self.school = self.r_api.create('school', {'name': 'DGN'})
self.program = self.r_api.create('program', {
'name': 'Test Program',
'abbreviation': 'TP1',
})
self.bad_program = self.r_api.create('program', {
'name': 'Bad Program',
'abbreviation': 'TP2',
})
self.cohort = self.r_api.create(
'cohort', {
'name': 'DGN 2014',
'code': 'trout viper',
'program': self.program.id,
'school': self.school.id,
})
self.bad_cohort = self.r_api.create(
'cohort', {
'name': 'bad cohort',
'code': 'king dolphin',
'program': self.bad_program.id,
'school': self.school.id,
})
self.r_api.associate('associate', self.teacher, self.cohort)
self.classroom = self.t_api.create(
'classroom', {
'name': "English 101",
'user': self.teacher.id,
'program': self.program.id,
'cohort': self.cohort.id,
})
# the researcher has to create this one, since the teacher is only
# associated with the "good" program.
self.bad_classroom = self.r_api.create(
'classroom', {
'name': "English 101",
'user': self.teacher.id,
'program': self.bad_program.id,
'cohort': self.bad_cohort.id,
})
# Identify a student into a classroom. See
# url_handlers._create_student()
self.student = self.p_api.create('user', {
'user_type': 'student',
'classroom': self.classroom.id,
})
self.student.put()
def test_get_user_by_query(self):
"""An example of an eventually consistent query."""
# A newly-created student won't be returned by a property-based query.
new_student = User.create(user_type='student')
new_student.put()
fetched_student = User.all().filter('id =', new_student.id).get()
self.assertEqual(new_student, fetched_student)
def immaculately_conceive(self):
"""Create and sign in to PERTS as a god user if no other user is signed
in and you are logged in to google as an app admin."""
user_id = self.session.get('user')
if not user_id and app_engine_users.is_current_user_admin():
google_user = app_engine_users.get_current_user()
god = User.all().filter('login_email =', google_user.email()).get()
if not god:
god = User.create(user_type='god', name=google_user.nickname(),
login_email=google_user.email(),
auth_id='google_' + google_user.user_id())
god.put()
self.session['user'] = god.id
def get(self, *args, **kwargs):
"""Handles ALL requests. Figures out the user and then delegates to
more purpose-specific methods."""
# Some poorly-behaved libraries screw with the default logging level,
# killing our 'info' and 'warning' logs. Make sure it's set correctly
# for our code.
util.allow_all_logging()
# Mark the start time of the request. You can add events to this
# profiler in any request handler like this:
# util.profiler.add_event("this is my event")
util.profiler.clear()
util.profiler.add_event('START')
# create and sign in as a god user under certain conditions
self.immaculately_conceive()
# Running this ensures that cached users (e.g. self._user) are
# correctly synched with the session ids.
self.clean_up_users('user')
self.clean_up_users('impersonated_user')
# Deal with impersonation options
if self.request.get('escape_impersonation') in config.true_strings:
# Allow calls to escape impersonation, i.e. treat the call as if
# the true user (not the impersonated user) was making it.
# Should have no effect when no impersonation is happening.
self._impersonated_user = None
self.api = Api(self.get_current_user(method='normal'))
elif self.request.get('impersonate'):
# If allowed, impersonate the provided user for just this call
target_user = User.get_by_id(self.request.get('impersonate'))
normal_user = self.get_current_user(method='normal')
if normal_user.can_impersonate(target_user):
self._impersonated_user = target_user
self.api = Api(target_user)
else:
logging.info('Impersonation denied. '
'{} is not allowed to impersonate {}.'
.format(normal_user.id, target_user.id))
if not hasattr(self, 'api'):
# Instantiate an api class that knows about the current user and
# their permissions. Will default to the permissions of the
# impersonated user if there is one, or the public user if no one
# is signed in.
self.api = Api(self.get_current_user())
# Also create an internal API that has god-like powers, only to be used
# by interal platform code, never exposed through URLs.
self.internal_api = Api(User.create(user_type='god'))
# We are in the "grandparent" function; do_wrapper() is the "parent",
# defined in either ApiHandler or ViewHandler; in turn, the "child"
# function do() will be called.
self.do_wrapper(*args, **kwargs)
def test_lower_case_login(self):
"""Check that the mapper function properly changes all possible values
of a user's auth_id and login_email."""
num_users = 100
email_length = 20
for x in range(num_users):
# Generate a random user, sometimes with string data, sometimes
# with None.
chars = string.digits + string.letters + string.punctuation
rand_str = ''.join(random.choice(chars) for x in range(20))
email = rand_str + '@a.aa' if random.random() > 0.5 else None
auth_id = 'direct_' + rand_str if random.random() > 0.5 else None
user = User(login_email=email, auth_id=auth_id)
mapper = map_module.LowerCaseLoginMapper
# Set up a fake job context for the mapper
conf = map_module.modify_kind('user', mapper, submit_job=False)
context = map_module.get_fake_context(conf)
# Manipulate the user
user = mapper().do(context, user)
# Check that the user has been manipulated properly
if user.auth_id is not None:
self.assertEquals(user.auth_id, user.auth_id.lower())
if user.login_email is not None:
self.assertEquals(user.login_email, user.login_email.lower())
def _base_identify_query(self, classroom_id):
"""The various ways we look for students have these similarities."""
query = User.all().filter('deleted =', False)
query.filter('is_test =', False)
query.filter('user_type =', 'student')
query.filter('assc_classroom_list =', classroom_id)
return query
def show_reminders(self, date_string=None):
god_api = Api(User(user_type='god'))
cron = Cron(god_api)
reminders = cron.get_reminders_by_date(date_string)
# Add a preview of how the text will be converted to html when sent.
for r in reminders:
r['html'] = markdown.markdown(r['body'])
return reminders
def do(self):
# Make sure only public users make this call.
user = self.get_current_user()
if user.user_type != 'public':
# Warn the javascript on the page that
# there's a problem so it can redirect.
return {'success': True, 'data': 'logout_required'}
params = util.get_request_dictionary(self.request)
# If there's a javascript stand-in for the calendar, there will be an
# extraneous parameter that's just for display; remove it.
if 'display_birth_date' in params:
del params['display_birth_date']
# The client supplies the classroom id, but we want the entity.
classroom = Classroom.get_by_id(params['classroom'])
params['classroom'] = classroom
# user may not be there, so check separately
user_id = self.request.get('user')
if user_id:
# the user has selected this among partial matches as themselves
# check the id; if it's valid, log them in
User.get_by_id(user_id) # an invalid id will raise errors
self.session['user'] = user_id
data = {
'exact_match': user_id,
'partial_matches': [],
'new_user': False
}
else:
# the user has provided identifying info, attempt to look them up
# see url_handlers.BaseHandler.identify() for structure of the
# data returned
data = self.identify(**params)
return {'success': True, 'data': data}
def batch_put_user(self, params):
"""Put many users.
See api_handlers.BatchPutUserHandler for structure of params.
"""
users = []
# If you look at api_handlers.BatchPutUserHandler you can see the
# structure of the params dictionary. It's got all the normal
# parameters to create a user, but instead of the user's first name
# and last name, it has a list of such names, representing many users
# to create.
# Remove the list to convert the batch parameters to creation
# parameters for a single user (which are now missing name arguments,
# obviously). We'll fill in names one by one as we create users.
user_names = params['user_names'] # dicts w/ 'first_name', 'last_name'
del params['user_names']
# Make one user through the normal api as a template, which will raise
# any relevant configuration or permissions exceptions. After this
# single create() call, we'll skip all that fancy logic in favor of
# speed.
template_user_params = params.copy()
template_user_params.update(user_names.pop())
template_user = self.create('user', template_user_params)
# Make all the other users in memory, copying relationships from the
# template. First we'll need to determine which are the relationship-
# containing properties.
relationship_property_names = [
p for p in dir(User) if p.split('_')[0] in ['assc', 'owned']
]
for user_info in user_names:
loop_params = params.copy()
loop_params.update(user_info)
loop_user = User.create(**loop_params) # doesn't put()
for prop in dir(loop_user):
if prop in relationship_property_names:
template_value = getattr(template_user, prop)
setattr(loop_user, prop, template_value)
users.append(loop_user)
# Save all the users in a single db operation.
db.put(users)
# Put the template user in the list so they're all there.
users.append(template_user)
return users
def cross_site_test(self, code, user_id):
"""Checks whether Qualtrics has successfully recorded a pd."""
user = User.get_by_id(user_id)
logging.info("Test user: {}".format(user))
if not user:
return False
pd_list = self.internal_api.get('pd', {'variable': 'cross_site_test'},
ancestor=user)
logging.info("Found pd: {}".format(pd_list))
if len(pd_list) != 1:
return False
pd = pd_list[0]
logging.info("Matching pd code. Looking for {}, found {}.".format(
code, pd.value))
return pd.value == code
def _disassociate(self, id, cache={}):
"""Find all users associated with this entity and remove the
relationship. Does not touch the datastore."""
logging.info('Api._disassociate(id={})'.format(id))
kind = core.get_kind(id)
for relation in ['assc', 'owned']:
prop = '{}_{}_list'.format(relation, kind)
query = User.all().filter('deleted =', False)
query.filter(prop + ' =', id)
for entity in query.run():
if entity.id in cache:
# prefer the cached entity over the queried one b/c the
# datastore is eventually consistent and repeatedly getting
# the same entity is not guaranteed to reflect changes.
entity = cache[entity.id]
else: # cache the entity so it can be used again
cache[entity.id] = entity
relationship_list = getattr(entity, prop)
relationship_list.remove(id)
return cache
def check_reset_password_token(self, token_string):
"""Validate a token supplied by a user.
Returns the matching user entity if the token is valid.
Return None if the token doesn't exist or has expired.
"""
token_entity = ResetPasswordToken.get_by_id(token_string)
if token_entity is None:
# This token doesn't exist. The provided token string is invalid.
return None
# Check that it hasn't expired and isn't deleted
one_hour = datetime.timedelta(hours=1)
expired = datetime.datetime.now() - token_entity.created > one_hour
if expired or token_entity.deleted:
# Token is invalid.
return None
return User.get_by_id(token_entity.user)
def get_current_user(self, method=None):
"""Get the currently logged in PERTS user with the following priority:
1. Impersonated user
2. Normal user
3. Public user
The method argument can override this behavior. If overriding and the
requested type of user is not present, will return the public user.
"""
public_user = User(user_type='public')
# Check that the session matches the cached user entites.
self.clean_up_users('user')
self.clean_up_users('impersonated_user')
# return what was asked
if method == 'normal':
return self._user or public_user
elif method == 'impersonated':
return self._impersonated_user or public_user
else:
return self._impersonated_user or self._user or public_user
def clean_up_users(self, session_key):
"""Brings the three representations of users into alignment: the entity
in the datastore, the id in the session, and the cached object saved as
a property of the request handler."""
id = self.session.get(session_key)
attr = '_' + session_key
cached_user = getattr(self, attr)
if not id:
# clear the cached user b/c the session is empty
setattr(self, attr, None)
elif not cached_user or id != cached_user.id:
# the cached user is invalid, try to restore from the session...
datastore_user = User.get_by_id(id)
if datastore_user:
# found the user defined by the session; cache them
setattr(self, attr, datastore_user)
else:
# could NOT find the user; clear everything
del self.session[session_key]
setattr(self, attr, None)
# Make sure the session keys always exist, even if they are empty.
if session_key not in self.session:
self.session[session_key] = None
def aggregate_to_users(self):
"""Calculate session progress and accounted_for status for users.
'progress' is just a redundant storage of a student's progress pd
value. This makes it easier to display the progress of many students at
once without having to pull pd.
"""
# Some terminology:
# changed_X: these things are around because the aggregator detected
# they have recent modifications; they come from
# aggregator.get_changed(kind).
# referenced_X: these things are referenced by things that have
# changed. Often, they're around because we need to "back-query"
# stuff to get complete totals. They're not necessarily changed,
# but we have to roll them into our statistics because they're
# siblings of things that have changed, and we're summarizing all
# the children into the parent.
# Trigger 1 of 2: modified pds which need to aggregate to their users.
# E.g. a student's progress pd has increased.
util.profiler.add_event('...get changed pd')
changed_pds = self.get_changed('pd')
# We only want user-related progress pds which have an activity ordinal
# (some testing pds don't have an ordinal). Do some filtering.
util.profiler.add_event('...process')
referenced_user_ids = []
changed_progress_pds = []
for pd in changed_pds:
is_user_pd = core.get_kind(pd.scope) == 'user'
has_ordinal = isinstance(pd.activity_ordinal, (int, long))
if pd.is_progress() and is_user_pd and has_ordinal:
if pd.scope not in referenced_user_ids:
referenced_user_ids.append(pd.scope)
changed_progress_pds.append(pd)
if len(referenced_user_ids) > 0:
referenced_users = User.get_by_id(referenced_user_ids)
else:
referenced_users = []
# Trigger 2 of 2: modified users whose status codes may have changed.
# E.g. a student is marked absent, "accounting for" their lack of
# participation.
util.profiler.add_event('...get changed users')
changed_users = self.get_changed('user')
# Unique list of users from both triggers.
changed_users = list(set(changed_users + referenced_users))
# Most aggregation runs will have nothing new to aggregate. Exit
# quickly to save cpu load.
if len(changed_users) is 0:
return []
# Now with all the data in hand, start summarizing it and storing it
# in user entities.
util.profiler.add_event('...process')
pds_by_user = util.list_by(changed_progress_pds, 'scope')
for user in changed_users:
if user.id in pds_by_user:
# Under normal operation, there should only be one active pd
# per user per activity ordinal. But we've found evidence that
# there may be several, due to inconsistent querying in
# self.get_changed(). If there are multiple that might cause
# incorrect aggregation, log an error, and
# make sure to choose the highest progress value available.
pds = pds_by_user[user.id]
pds_by_ordinal = util.list_by(pds, 'activity_ordinal')
if any([len(v) > 1 for v in pds_by_ordinal.values()]):
logging.warning(
"Multiple pds found in aggregation: {}".format(
json.dumps([pd.to_dict() for pd in pds])))
agg_data = {}
for pd in pds:
# Record progress values by activity ordinal
# Why coerce to int here? Sometimes it's a long.
# https://cloud.google.com/appengine/docs/python/datastore/typesandpropertyclasses#IntegerProperty
o = int(pd.activity_ordinal)
user.aggregation_data.setdefault(
o, copy.deepcopy(user.aggregation_data_template))
# v = int(pd.value)
# user.aggregation_data[o]['progress'] = v
# if v is 100 and user.get_status_code(o) is None:
new_v = int(pd.value)
# Keep track of which ordinals we've seen before
if o in agg_data and new_v < agg_data[o]['progress']:
# See pull #306
logging.error(
"Out-of-order hypothesis confirmed! {}".format(
pds_by_user[user.id]))
elif o not in agg_data:
agg_data[o] = {'progress': None}
# Only save the value if it's larger than the previous
# (only relevant when there are more than one).
current_v = agg_data[o]['progress']
if current_v is None or new_v > current_v:
agg_data[o]['progress'] = new_v
if new_v is 100 and user.get_status_code(o) is None:
# Also assign the status code "Completed" to this
# student. This is technically redundant, but makes
# the data clearer.
user.set_status_code(o, 'COM')
# Copy compiled results into the user.
for k, v in agg_data.items():
user.aggregation_data[k] = agg_data[k]
# Save changes to users and the aggregator timestamp.
util.profiler.add_event('...save users')
self.save('user', changed_users)
return changed_users
def test_get_user_by_key(self):
"""An example of a strongly consistent query."""
fetched_student = User.get_by_id(self.student.id)
self.assertEqual(self.student, fetched_student)
class ProgramAppTestCase(unit_test_helper.PertsTestCase):
def set_up(self):
# Load our page handler webapp and wrap it with WebTest's TestApp.
self.testapp = webtest.TestApp(page_handlers.app)
# db with well-behaved strong consistency
self.testbed.init_datastore_v3_stub()
self.init_context()
def init_context(self):
# public api
self.p_api = Api(User(user_type='public'))
# This closely follows the pattern of the populate script in god.html
self.researcher = User(user_type='researcher')
self.researcher.put()
self.r_api = Api(self.researcher)
self.teacher = User.create(user_type='teacher')
self.teacher.put()
self.t_api = Api(self.teacher)
self.school = self.r_api.create('school', {'name': 'DGN'})
self.program = self.r_api.create('program', {
'name': 'Test Program',
'abbreviation': 'TP1',
})
self.bad_program = self.r_api.create('program', {
'name': 'Bad Program',
'abbreviation': 'TP2',
})
self.cohort = self.r_api.create(
'cohort', {
'name': 'DGN 2014',
'code': 'trout viper',
'program': self.program.id,
'school': self.school.id,
})
self.bad_cohort = self.r_api.create(
'cohort', {
'name': 'bad cohort',
'code': 'king dolphin',
'program': self.bad_program.id,
'school': self.school.id,
})
self.r_api.associate('associate', self.teacher, self.cohort)
self.classroom = self.t_api.create(
'classroom', {
'name': "English 101",
'user': self.teacher.id,
'program': self.program.id,
'cohort': self.cohort.id,
})
# the researcher has to create this one, since the teacher is only
# associated with the "good" program.
self.bad_classroom = self.r_api.create(
'classroom', {
'name': "English 101",
'user': self.teacher.id,
'program': self.bad_program.id,
'cohort': self.bad_cohort.id,
})
# Identify a student into a classroom. See
# url_handlers._create_student()
self.student = self.p_api.create('user', {
'user_type': 'student',
'classroom': self.classroom.id,
})
self.student.put()
@unittest.skip("Throwing errors in production. Issue #260.")
def test_malformed_urls(self):
correct_urls = [
'/p/TP1/teacher?cohort={}'.format(self.cohort.id),
'/p/TP1/student?cohort={}&classroom={}'.format(
self.cohort.id, self.classroom.id),
]
for url in correct_urls:
response = self.testapp.get(url)
self.assertEqual(response.status_int, 200, msg=url)
# Now we test a bunch of urls that return 404s. 404s in webtest raises
# an exception, which we can assert.
malformed_urls = [
# urls with no arguments shouldn't work
'/p/TP1/student',
'/p/TP1/teacher',
# student missing classroom
'/p/TP1/student?cohort={}'.format(self.cohort.id),
# student missing cohort
'/p/TP1/student?classroom={}'.format(self.classroom.id),
# teacher, mis-associated cohort
'/p/TP1/teacher?cohort={}'.format(self.bad_cohort.id),
# student, mis-associated cohort
'/p/TP1/teacher?cohort={}&classroom={}'.format(
self.bad_cohort.id, self.classroom.id),
# student, mis-associated classroom
'/p/TP1/student?cohort={}&classroom={}'.format(
self.cohort.id, self.bad_classroom.id),
# teacher, non-existent cohort
'/p/TP1/teacher?cohort=DNE',
# student, non-existent cohort
'/p/TP1/student?cohort={}&classroom={}'.format(
'DNE', self.classroom.id),
# student, non-existent classroom
'/p/TP1/student?cohort={}&classroom={}'.format(
self.cohort.id, 'DNE'),
# student, both non-existent
'/p/TP1/student?cohort={}&classroom={}'.format('DNE', 'DNE'),
]
for url in malformed_urls:
self.assertRaises(webtest.AppError, self.testapp.get, url)
def populate(self):
"""Creates a standard set of entities as PERTS expects them to exist.
Includes all the correct relationships, and leaves everything in a
consistent state. Mimics the javascript populate script in god.html.
"""
self.public_api = Api(User(user_type='public'))
self.internal_api = Api(User(user_type='god'))
self.researcher = User.create(user_type='researcher')
self.researcher_api = Api(self.researcher)
self.researcher.put()
self.school = self.researcher_api.create('school', {'name': 'DGN'})
self.program = self.researcher_api.create('program', {
'name': 'Test Program',
'abbreviation': 'TP1',
})
self.cohort = self.researcher_api.create(
'cohort', {
'name': 'DGN 2014',
'code': 'trout viper',
'program': self.program.id,
'school': self.school.id,
})
self.school_admin = self.internal_api.create('user', {
'user_type': 'school_admin',
'program': self.program.id
})
self.school_admin_api = Api(self.school_admin)
# have the researcher set the school_admin as an owner of their cohort
self.researcher_api.associate('set_owner', self.school_admin,
self.cohort)
self.teacher = self.public_api.create('user', {
'user_type': 'teacher',
'program': self.program.id
})
self.teacher.put()
self.teacher_api = Api(self.teacher)
self.researcher_api.associate('associate', self.teacher, self.cohort)
# This is normally done api_handlers.AssociateHandler
self.teacher_activities = self.teacher_api.init_activities(
'teacher',
self.teacher.id,
self.program.id,
cohort_id=self.cohort.id)
# school_admins create classrooms.
self.classroom = self.school_admin_api.create(
'classroom', {
'name': "English 101",
'user': self.teacher.id,
'program': self.program.id,
'cohort': self.cohort.id,
})
self.student = self.public_api.create('user', {
'user_type': 'student',
'classroom': self.classroom.id,
})
self.student.put()
self.student_api = Api(self.student)
# This is normally done api_handlers.CreateHandler
# school_admins create classrooms (and thus student activities).
self.student_activities = self.school_admin_api.init_activities(
'student',
self.teacher.id,
self.program.id,
cohort_id=self.cohort.id,
classroom_id=self.classroom.id)
# Force everything into a consistent state, just in case we're using
# an inconsistent policy.
db.get([
self.researcher.key(),
self.school_admin.key(),
self.teacher.key(),
self.student.key(),
self.school.key(),
self.program.key(),
self.cohort.key(),
self.classroom.key(), self.teacher_activities[0].key(),
self.teacher_activities[1].key(), self.student_activities[0].key(),
self.student_activities[1].key()
])
def aggregate_to_activities(self, changed_users):
"""Calculate basic stats of users in an activity for reporting.
For instance, a count of all users with progress 100 will be saved
with the activity as aggregation_data['all']['completed']."""
# Firm assumptions: these are enforced.
# 1. We only want to aggregate student-users to student-type activities
# Soft assumptions: violating these assumptions will log errors but
# will not break the aggregator.
# 1. All students have one associated classroom.
changed_students = []
students_breaking_assumptions = []
for user in changed_users:
if user.user_type == 'student':
if len(user.assc_classroom_list) is 1:
changed_students.append(user)
else:
students_breaking_assumptions.append(user)
if len(students_breaking_assumptions) > 0:
logging.error(
"Students with bad classroom associations: {}".format(
students_breaking_assumptions))
# Set up an index of users and activities by classroom (we'll need this
# later).
index = {}
cohort_ids = [] # Get list of unique cohorts.
for s in changed_students:
cl_id = s.assc_classroom_list[0]
if cl_id not in index:
index[cl_id] = {'students': [], 'activities': []}
co_id = s.assc_cohort_list[0]
if co_id not in cohort_ids:
cohort_ids.append(co_id)
classroom_ids = index.keys()
if len(classroom_ids) is 0:
return []
# Activities don't store relationships with users directly. To find
# them, query via classroom ids.
util.profiler.add_event('...get related activities')
query = Activity.all().filter('deleted =', False)
query.filter('is_test =', False)
query.filter('user_type =', 'student')
# If there are more than 30, don't filter by classroom id directly,
# because App Engine limits subqueries. Instead, filter the query by
# cohort, then further filter by classroom in-memory.
if len(classroom_ids) < 30:
logging.info(
"Using normal classroom query for classroom activities")
query.filter('assc_classroom_list IN', classroom_ids)
fetched_activities = query.run()
else:
logging.info(
"> 30 classrooms: Using cohort query for classroom activities")
query.filter('assc_cohort_list IN', cohort_ids)
fetched_activities = [
a for a in query.run()
if a.assc_classroom_list[0] in classroom_ids
]
# Soft assumptions: violating these assumptions will log errors but
# will not break the aggregator.
# 1. All student activities have one associated classroom.
util.profiler.add_event('...process')
referenced_activities = []
activities_breaking_assumptions = []
for a in fetched_activities:
if len(a.assc_classroom_list) is 1:
referenced_activities.append(a)
else:
activities_breaking_assumptions.append(a)
if len(activities_breaking_assumptions) > 0:
logging.error(
"Activities with bad classroom associations: {}".format(
activities_breaking_assumptions))
# Add the activities to the index.
for a in referenced_activities:
c_id = a.assc_classroom_list[0]
index[c_id]['activities'].append(a)
# Re-query users for these classrooms so we can calculate the total
# number of students per classroom not just the change in number of
# students per classroom.
util.profiler.add_event('...get related users')
query = User.all().filter('deleted =', False)
query.filter('is_test =', False)
query.filter('user_type =', 'student')
# If there are more than 30, don't filter by classroom id directly,
# because App Engine limits subqueries. Instead, filter the query by
# cohort, then further filter by classroom in-memory.
if len(classroom_ids) < 30:
logging.info("Using normal classroom query for activity users")
query.filter('assc_classroom_list IN', classroom_ids)
referenced_students = query.run()
else:
logging.info(
"> 30 classrooms: using cohort query for activity users")
query.filter('assc_cohort_list IN', cohort_ids)
referenced_students = [
u for u in query.run()
if u.assc_classroom_list[0] in classroom_ids
]
# Add the students to the index.
util.profiler.add_event('...process')
changed_student_index = {s.id: s for s in changed_students}
for s in referenced_students:
# Important: some of these users *just changed*, i.e. just had
# their aggregation modified and have been passed in to this
# function as changed_students. They may be more up to date than
# the version of the same entity returned by they query.
if s.id in changed_student_index:
# So, when possible, prefer the already-in-memory version.
s = changed_student_index[s.id]
# If any of the student's codes indicate that they somehow aren't
# a real student (e.g. marked "Discard"), don't count them at all.
exclude_student = False
for ordinal, code in s.status_codes.items():
if code and config.status_codes[code]['exclude_from_counts']:
exclude_student = True
if exclude_student:
# Don't do anything with this student, continue to the next one
continue
c_id = s.assc_classroom_list[0]
index[c_id]['students'].append(s)
# Iterate over activities, calculating stats based on the related set
# of users.
# See the summarize_students() method and/or the docs:
# https://docs.google.com/document/d/1tmZhuWMDX29zte6f0A8yXlSUvqluyMNEnFq8qyJq1pA
for classroom_id, d in index.items():
for a in d['activities']:
a.aggregation_data['total_students'] = len(d['students'])
cert_students = [s for s in d['students'] if s.certified]
a.aggregation_data['certified_students'] = len(cert_students)
# Why coerce to int here? Sometimes it's a long.
# https://cloud.google.com/appengine/docs/python/datastore/typesandpropertyclasses#IntegerProperty
s = self.summarize_students(cert_students,
int(a.activity_ordinal))
a.aggregation_data['certified_study_eligible_dict'] = s
util.profiler.add_event('...save activities')
self.save('activity', referenced_activities)
return referenced_activities
def test_deidentify(self):
"""Check that the mapper function properly hashes requested users."""
# When running this for real, a secret random salt will be specified
# by the adminstrator issuing the job. For this test, we'll use a
# dummy value
salt = u'salt'
# Generate two (different) random cohort ids
id1 = id2 = ''
while id1 == id2:
id1 = Cohort.generate_id(phrase.generate_phrase())
id2 = Cohort.generate_id(phrase.generate_phrase())
# Set up each way a user could be associated with an the cohort.
loner = User( # "loner" b/c no cohort associations
first_name=u"William",
last_name=u"Clinton",
login_email=u"",
stripped_first_name=util.clean_string(u"William"),
stripped_last_name=util.clean_string(u"Clinton"),
name=u"William",
birth_date=datetime.date(1946, 8, 19),
auth_id="",
title="President",
phone="(202) 456-1111",
notes="This is Bill Clinton.",
user_type="student",
)
standard = User( # "standard" b/c one cohort association
first_name=u"George",
last_name=u"Bush",
middle_initial='P',
login_email=u"",
stripped_first_name=util.clean_string(u"George"),
stripped_last_name=util.clean_string(u"Bush"),
name=u"George",
birth_date=datetime.date(1946, 7, 6),
auth_id="",
title="President",
phone="(202) 456-1111",
notes="This is George Bush Jr.",
assc_cohort_list=[id1],
user_type="student",
)
dual = User( # "dual" b/c two cohort associations
first_name=u"Ban Ki-moon",
last_name=u"\uBC18\uAE30\uBB38",
middle_initial='P',
login_email=u"",
stripped_first_name=util.clean_string(u"Ban Ki-moon"),
stripped_last_name=util.clean_string(u"\uBC18\uAE30\uBB38"),
name=u"Ban",
birth_date=datetime.date(1944, 6, 13),
auth_id="google_123445345738",
title="Secretary General",
phone="(212) 963 1234",
notes="This is Ban Ki-moon.",
assc_cohort_list=[id1, id2],
user_type="student",
)
adult = User( # "adult" b/c user type teacher
first_name=u"Barack",
last_name=u"Obama",
middle_initial='P',
login_email=u"",
stripped_first_name=util.clean_string(u"Barack"),
stripped_last_name=util.clean_string(u"Obama"),
name=u"Barack",
birth_date=datetime.date(1961, 8, 4),
auth_id="",
title="President",
phone="(202) 456-1111",
notes="This is Barack Obama.",
assc_cohort_list=[id1],
user_type="teacher",
)
# Set up a fake job context for the mapper, requesting that all users
# associated with the first cohort be deleted.
conf = map_module.deidentify('assc_cohort_list', [id1],
salt,
submit_job=False)
context = map_module.get_fake_context(conf)
mapper = map_module.DeidentifyMapper()
# Manipulate each user
deidentified_loner = mapper.do(context, copy.deepcopy(loner))
deidentified_standard = mapper.do(context, copy.deepcopy(standard))
deidentified_dual = mapper.do(context, copy.deepcopy(dual))
deidentified_adult = mapper.do(context, copy.deepcopy(adult))
# Check that users not specified are not modified.
self.assertEqual(loner, deidentified_loner)
# Check that non-students are unchanged, even if they have the right
# relationship.
self.assertEqual(adult, deidentified_adult)
# With modified users, these properties should be erased i.e. set to ''
erased_properties = [
'stripped_first_name', 'stripped_last_name', 'name', 'auth_id',
'title', 'phone', 'notes', 'auth_id', 'middle_initial'
]
self.assertEqual(deidentified_standard.first_name,
mapper.hash(u"George", salt))
self.assertEqual(deidentified_standard.last_name,
mapper.hash(u"Bush", salt))
self.assertIsNone(deidentified_standard.login_email,
mapper.hash(u"", salt))
self.assertEqual(deidentified_standard.birth_date,
datetime.date(1946, 7, 1))
for p in erased_properties:
self.assertEqual(getattr(deidentified_standard, p), '')
self.assertEqual(deidentified_dual.first_name,
mapper.hash(u"Ban Ki-moon", salt))
self.assertEqual(deidentified_dual.last_name,
mapper.hash(u"\uBC18\uAE30\uBB38", salt))
self.assertIsNone(deidentified_dual.login_email,
mapper.hash(u"", salt))
self.assertEqual(deidentified_dual.birth_date,
datetime.date(1944, 6, 1))
for p in erased_properties:
self.assertEqual(getattr(deidentified_dual, p), '')
# If we run the process again, nothing should change b/c the job
# should be idempotent.
final_loner = mapper.do(context, copy.deepcopy(deidentified_loner))
final_standard = mapper.do(context,
copy.deepcopy(deidentified_standard))
final_dual = mapper.do(context, copy.deepcopy(deidentified_dual))
final_adult = mapper.do(context, copy.deepcopy(deidentified_adult))
self.assertEqual(final_loner, deidentified_loner)
self.assertEqual(final_standard, deidentified_standard)
self.assertEqual(final_dual, deidentified_dual)
self.assertEqual(final_adult, deidentified_adult)
def authenticate(self, auth_type=None, username=None, password=None):
"""Takes various kinds of credentials (username/password, google
account) and logs you in as a PERTS user.
Will return one of these three:
User object: the user has been successfully authenticated
False: credentials invalid, either because a password is wrong
or no account exists for those credentials
None: looked for credentials but didn't find any of the
appropriate kind
"""
if auth_type is None:
# determine auth_type if it wasn't provided
if username is not None and password is not None:
auth_type = 'direct'
elif app_engine_users.get_current_user():
auth_type = 'google'
else:
# Auth_type might still be None, which simply means that
# someone has arrived at login but we don't have any data on
# them yet. Don't bother with the rest of the function.
return None
# fetch matching users
if auth_type == 'direct':
if username is None or password is None:
# no credentials present
return None
auth_id = 'direct_' + username.lower()
user_query_lowercase = User.all().filter('deleted =', False)
user_query_lowercase.filter('is_test =', False)
user_query_lowercase.filter('auth_id =', auth_id)
user_results = user_query_lowercase.fetch(2)
elif auth_type in ['google']:
try:
user_kwargs = self.get_third_party_auth(auth_type)
except CredentialsMissing:
return None
auth_id = user_kwargs['auth_id']
user_query = User.all().filter('deleted =', False)
user_query.filter('auth_id =', auth_id)
# Fetch 2 b/c that's sufficient to detect multiple matching users.
user_results = user_query.fetch(2)
# interpret the results of the query
num_matches = len(user_results)
if num_matches is 0:
# no users with that username, invalid log in
return False
elif num_matches > 1:
logging.error("More than one user matches auth info: {}"
.format(auth_id))
# Sort the users by modified time, so we take the most recently
# modified one.
user_results = sorted(
user_results, key=lambda u: u.modified, reverse=True)
# else num_matches is 1, the default case, and we can assume there was
# one matching user
user = user_results[0]
# For direct authentication, PERTS is in charge of
# checking their credentials, so validate the password.
if auth_type == 'direct':
# A user-specific salt AND how many "log rounds" (go read about key
# stretching) should be used is stored IN the user's hashed
# password; that's why it's an argument here.
# http://pythonhosted.org/passlib/
if not sha256_crypt.verify(password, user.hashed_password):
# invalid password for this username
return False
# We've decided our response; if the user has successfully
# authenticated, then log them in to the session.
if isinstance(user, User):
# all's well, log them in and return the matching user
self.session['user'] = user.id
return user