def root(self):
session = self.request.session if self.request.session else None
user = self.request.user if self.request.user else None
profiles = None
emails = None
if user:
profile_keys = [ndb.Key("UserProfile", p) for p in user.auth_ids]
profiles = ndb.get_multi(profile_keys)
emails = models.UserEmail.get_by_user(user.key.id())
self.render_template("home.html", {"user": user, "session": session, "profiles": profiles, "emails": emails})
def handle(self):
start = self.get_required('start', int, 0)
end = self.get_required('end', int, 0)
count = self.get_required('count', int, 100)
query = models.PostReference.query()
if not start and not end:
# Get newest posts by doing a prefix scan.
start_key = ndb.Key(
models.Shard._get_kind(), self.shard,
models.PostReference._get_kind(), 1)
end_key = ndb.Key(
models.Shard._get_kind(), self.shard,
models.PostReference._get_kind(), 2**62)
elif not start and end:
# Get a specific set of posts
start_key = ndb.Key(
models.Shard._get_kind(), self.shard,
models.PostReference._get_kind(), max(1, end - count))
end_key = ndb.Key(
models.Shard._get_kind(), self.shard,
models.PostReference._get_kind(), end)
else:
# Should not happen
assert False
query = query.filter(models.PostReference.key >= start_key)
query = query.filter(models.PostReference.key <= end_key)
query = query.order(-models.PostReference.key)
ref_list = query.fetch(count)
post_kind = models.Post._get_kind()
post_key_list = [ndb.Key(post_kind, ref.post_id) for ref in ref_list]
post_list = ndb.get_multi(post_key_list)
adjusted_post_list = []
for post, ref in zip(post_list, ref_list):
# PostReference entities may point to non-existent Post entities
# once the cleanup job has run. Filter them out here. The client
# side won't try to scan for posts previous to the last one that's
# actually found, so this filtering is okay.
if not post:
continue
# TODO(bslatkin): Drop post entities that are older than
# config.ephemeral_lifetime_seconds, in case the background job
# takes longer than expected to delete things.
post.sequence = ref.sequence
adjusted_post_list.append(post)
self.json_response['posts'] = marshal_posts(
self.shard, adjusted_post_list)
def post(self):
email = self.request.get('email')
if not email:
# validate email address
# TODO: use WTForm
self.add_message('Please provide an email address', 'error')
return self.redirect_to('password-reset')
# validate existence of email address
auth_id = models.User.generate_auth_id('email', email)
user = models.User._find_user(auth_id, [email])
if not user:
self.add_message('No one with that email address was found', 'error')
return self.redirect_to('password-reset')
profiles = ndb.get_multi(ndb.Key(models.UserProfile, key) for key in user.auth_ids)
if not profiles:
self.add_message('No one with that email address was found', 'error')
return self.redirect_to('password-reset')
pwd = None
for profile in profiles:
if profile.password:
pwd = profile.password
if pwd:
# generate and save reset_token
timestamp = str(time.time())
token_value = security.generate_password_hash(pwd+timestamp, length=12)
callback_uri = "%(uri)s?ts=%(ts)s&token=%(token)s&email=%(email)s" %dict(uri=webapp2.uri_for('new-password', _full=True),
ts=timestamp, email=email, token=token_value)
# send validation email
from google.appengine.api import mail
mail.send_mail(sender="Example.com Support <*****@*****.**>",
to=email,
subject="Password reset",
body="""
Dear %(email)s:
Click on the following link to reset your password %(callback)s
Please let us know if you have any questions.
The example.com Team
"""%dict(email=email, callback=callback_uri))
self.add_message('We sent you an email with instructions to reset your password')
else:
self.add_message('You previously associated %(providers)s' %dict(providers=', '.join([profile.key.id() for profile in profiles])))
return self.redirect_to('password-reset')
def txn():
read_state_list = ndb.get_multi(read_state_keys)
to_put = []
for key, read_state in zip(read_state_keys, read_state_list):
next_read_sequence = topic_dict[key.id()]
if read_state is None:
read_state = models.ReadState(key=key)
read_state.last_read_sequence = next_read_sequence
else:
read_state.last_read_sequence = max(
read_state.last_read_sequence, next_read_sequence)
to_put.append(read_state)
ndb.put_multi(to_put)
def get(self):
profiles = None
emails = None
user = self.user
if user:
profiles_done = [p[:p.index(':')] for p in user.auth_ids]
profile_keys = [ndb.Key('UserProfile', p) for p in user.auth_ids]
profiles = ndb.get_multi(profile_keys)
emails = models.UserEmail.get_by_user(user.key.id())
self.render_template('profile.html', {
'profiles': profiles,
'profiles_done': profiles_done,
'emails': emails,
'form': self.form,
})
def enqueue_email_tasks(emails_set):
"""Enqueues a set of email notification tasks for the given users.
Notifies these users across multiple shards.
Args:
emails_set: Set of user email addresses to notify.
"""
if not emails_set:
logging.debug('No email addresses to notify')
return
logging.debug('Found %d email addresses to notify', len(emails_set))
email_record_keys = [
ndb.Key(models.EmailRecord._get_kind(), email_address)
for email_address in emails_set]
email_record_list = ndb.get_multi(email_record_keys)
task_list = []
for email_address, email_record in zip(emails_set, email_record_list):
sequence_number = 1
countdown = config.default_notify_period_seconds
if email_record:
sequence_number = email_record.sequence_number
countdown = email_record.min_notify_period_seconds
task = taskqueue.Task(
url='/work/email_digest',
params=dict(sequence_number=sequence_number,
email_address=email_address),
name='email-notify-%s-%s' % (
models.human_hash(email_address), sequence_number),
countdown=countdown)
task_list.append(task)
logging.debug('Enqueuing email notification tasks for %d users',
len(task_list))
try:
taskqueue.Queue(config.email_digest_queue).add(task_list)
except (taskqueue.TombstonedTaskError, taskqueue.TaskAlreadyExistsError):
logging.debug('Enqueued email tasks for emails=%r; '
'at least one task already present', emails_set)
def apply_posts(shard=None,
insertion_post_id=None,
lease_seconds=10,
max_tasks=20):
"""Applies a set of pending posts to a shard.
If shard is None then this function will apply mods for whatever is the
first shard it can find in the pull task queue.
insertion_post_id is the post_id that first caused this apply task to be
enqueued. This task will retry until it applies the insertion_post_id
itself or it can confirm that the insertion_post_id has already been
applied. insertion_post_id may be empty if the apply task is not associated
with a particular post (such as cronjobs/cleanup tasks).
"""
# Do not use caching for NDB in this task queue worker.
ctx = ndb.get_context()
ctx.set_cache_policy(lambda x: False)
ctx.set_memcache_policy(lambda x: False)
# Fetch the new Posts to put in sequence.
queue = taskqueue.Queue(config.pending_queue)
# When no shard is specified, process the first tag we find.
task_list = []
if not shard:
task_list.extend(queue.lease_tasks(lease_seconds, 1))
if not task_list:
logging.debug('apply_posts with no specific shard found no tasks')
return
params = task_list[0].extract_params()
shard = params['shard']
logging.debug('apply_posts with no specific shard found shard=%r',
shard)
# Clear the dirty bit on this shard to start the time horizon.
dirty_bit(shard, clear=True)
# Find tasks pending for the current shard.
task_list.extend(
queue.lease_tasks_by_tag(lease_seconds, max_tasks, tag=str(shard)))
receipt_key_list = []
new_topic = None
for task in task_list:
params = task.extract_params()
# Extract the new topic shard associated with this task, if any. The
# last one wins. If all of the found posts have already been applied,
# then topic assignment will be ignored.
new_topic = params.get('new_topic') or new_topic
post_id_list = params.get('post_ids')
if post_id_list is None:
# This may happen on replica shards if it turns out there are no
# unapplied post IDs but an apply task still ran.
post_id_list = []
elif not isinstance(post_id_list, list):
post_id_list = [post_id_list]
for post_id in post_id_list:
receipt_key = ndb.Key(
models.Post._get_kind(), post_id,
models.Receipt._get_kind(), shard)
receipt_key_list.append(receipt_key)
receipt_list = ndb.get_multi(receipt_key_list)
# Some tasks may be in the pull queue that were already put in sequence.
# So ignore these and only apply the new ones.
unapplied_receipts = [
models.Receipt(key=k)
for k, r in zip(receipt_key_list, receipt_list)
if r is None]
unapplied_post_ids = [r.post_id for r in unapplied_receipts]
# Double check if we think there should be work to apply but we didn't find
# any. This will force the apply task to retry immediately if the post task
# was not found. This can happen when the pull queue's consistency is
# behind.
if not unapplied_receipts and insertion_post_id:
receipt_key = ndb.Key(
models.Post._get_kind(), insertion_post_id,
models.Receipt._get_kind(), shard)
receipt = receipt_key.get()
if receipt:
logging.warning(
'No post application to do for shard=%r, but post_id=%r '
'already applied; doing nothing in this task',
shard, insertion_post_id)
new_topic = None
# Do not 'return' here. We need to increment the shard sequence or
# else tasks will not run for this shard in the future because of
# de-duping.
else:
raise base.Error('No post application to do for shard=%r, but'
'post_id=%r has not been applied; will retry' %
(shard, insertion_post_id))
now = datetime.datetime.now()
def txn():
shard_record = models.Shard.get_by_id(shard)
# TODO(bslatkin): Just drop this task entirely if the shard cannot
# be found. Could happen for old shards that were cleaned up.
assert shard_record
# One of the tasks in this batch has a topic assignment. Apply it here.
if new_topic:
logging.debug('Changing topic from %r to %r',
shard_record.current_topic, new_topic)
shard_record.current_topic = new_topic
shard_record.topic_change_time = now
new_sequence_numbers = list(xrange(
shard_record.sequence_number,
shard_record.sequence_number + len(unapplied_receipts)))
shard_record.sequence_number += max(1, len(unapplied_receipts))
# Write post references that point at the newly sequenced posts.
to_put = [shard_record]
for receipt, sequence in zip(unapplied_receipts, new_sequence_numbers):
to_put.append(models.PostReference(
id=sequence,
parent=shard_record.key,
post_id=receipt.post_id))
# Update the receipt entity here; it will be written outside this
# transaction, since these receipts may span multiple entity
# groups.
receipt.sequence = sequence
# Enqueue replica posts transactionally, to make sure everything
# definitely will get copied over to the replica shard.
if shard_record.current_topic:
enqueue_post_task(shard_record.current_topic, unapplied_post_ids)
ndb.put_multi(to_put)
return shard_record, new_sequence_numbers
# Have this only attempt a transaction a single time. If the transaction
# fails the task queue will retry this task within 4 seconds. Because
# apply tasks are always named by the current Shard.sequence_number we
# can be reasonably sure that no other apply task for this shard will be
# running concurrently when this fails.
shard_record, new_sequence_numbers = ndb.transaction(txn, retries=1)
replica_shard = shard_record.current_topic
logging.debug('Applied %d posts for shard=%r, sequence_numbers=%r',
len(unapplied_receipts), shard, new_sequence_numbers)
futures = []
# Save receipts for all the posts.
futures.extend(ndb.put_multi_async(unapplied_receipts))
# Notify all logged in users of the new posts.
futures.append(notify_posts(
shard, unapplied_post_ids, sequence_numbers=new_sequence_numbers))
# Replicate posts to a topic shard.
if replica_shard:
logging.debug('Replicating source shard=%r to replica shard=%r',
shard, replica_shard)
futures.append(enqueue_apply_task(replica_shard))
# Success! Delete the tasks from this queue.
queue.delete_tasks(task_list)
# Always run one more apply task to clean up any posts that came in
# while this transaction was processing.
if dirty_bit(shard, check=True):
futures.append(enqueue_apply_task(shard))
# Wait on all pending futures in case they raise errors.
ndb.Future.wait_all(futures)
# For root shards, add shard cleanup task to check for user presence and
# cause notification of user logouts if the channel API did not detect the
# user closing the connection.
if not shard_record.root_shard:
presence.enqueue_cleanup_task(shard)
def handle_request(self, shard_id):
shard = models.Shard.get_by_id(shard_id)
if not shard:
# If the shard doesn't exist, then just create it. Makes it
# ridiculously easy for people to create a new chat with the name
# of their choice.
def txn():
shard = models.Shard.get_by_id(
shard_id, use_cache=False, use_memcache=False)
if not shard:
shard = models.Shard(id=shard_id)
shard.put()
return shard
shard = ndb.transaction(txn)
# Do not allow users to directly access topic shards.
if shard.root_shard:
# TODO(bslatkin): Make this pretty.
self.response.set_status(404)
return
nickname = 'Anonymous'
email_address = ''
first_login = True
must_accept_terms = True
sounds_enabled = True
if 'shards' in self.session:
# TODO(bslatkin): Reuse presence code here.
user_id = self.session['shards'].get(shard_id)
if user_id:
login_record = models.LoginRecord.get_by_id(user_id)
if login_record and login_record.shard_id == shard_id:
email_address = login_record.email_address
nickname = login_record.nickname
first_login = False
must_accept_terms = bool(
login_record.accepted_terms_version !=
config.terms_version)
sounds_enabled = login_record.sounds_enabled
else:
# Pre-populate the user's settings info from the newest other
# shard the user has.
all_user_ids = self.session['shards'].values()
all_login_records = ndb.get_multi([
ndb.Key(models.LoginRecord._get_kind(), user_id)
for user_id in all_user_ids])
all_login_records = [r for r in all_login_records if r]
all_login_records.sort(key=lambda x: x.last_update_time)
if all_login_records:
login_record = all_login_records[-1]
email_address = login_record.email_address
nickname = login_record.nickname
sounds_enabled = login_record.sounds_enabled
context = {
'email_address': email_address or '',
'first_login': first_login,
'must_accept_terms': must_accept_terms,
'nickname': xml.sax.saxutils.unescape(nickname),
'page_name': 'chatroom',
'shard_id': shard_id,
'short_url': self.request.path_url,
'sounds_enabled': sounds_enabled,
'xsrf_token': self.session['xsrf_token'],
}
context['params'] = json.dumps(context)
self.render('chatroom.html', context)