def sum_add(instance, field, aggregator_instance, aggregator_field, process_func=None):
"""
Shortcut function for adding sum operation. Requires next arguments:
* instance - instance of aggregated model;
* field - name of field the sum is calculated by;
* aggregator_instance - instance of the model the aggregation created for;
* aggregator_field - name of the field of aggregator_instance which is AggregatedProperty;
* process_func - input value processing function, usually is a lambda function like lambda x: abs(x);
"""
if process_func is None:
process_func = lambda x:x
ndb.get_context().clear_cache()
if instance.key.id():
old_instance = instance.key.get()
else:
old_instance = None
sum_part = process_func(getattr(instance, field))
if old_instance:
if process_func(getattr(old_instance, field)) != sum_part:
sum_part -= process_func(getattr(old_instance, field))
else:
return
#import pdb; pdb.set_trace()
setattr(aggregator_instance,
aggregator_field,
getattr(aggregator_instance, aggregator_field) + sum_part)
def shell():
if (not os.environ.get('DATASTORE_APP_ID', None)
and not os.environ.get('DATASTORE_PROJECT_ID', None)):
raise ValueError(
'Must set either DATASTORE_APP_ID or DATASTORE_PROJECT_ID'
' environment variable.')
ndb.get_context().set_memcache_policy(False)
ndb.get_context().set_cache_policy(False)
# ndb will set the application ID.
application_id = os.environ['APPLICATION_ID']
id_resolver = datastore_pbs.IdResolver((application_id, ))
project_id = id_resolver.resolve_project_id(application_id)
banner = """ndb shell
Python %s
Project: %s
The ndb module is already imported.
""" % (sys.version, project_id)
imports = {
'ndb': ndb,
}
# set up the environment
os.environ['SERVER_SOFTWARE'] = 'Development (ndb_shell)/0.1'
sys.ps1 = '%s> ' % project_id
if readline is not None:
# set up readline
readline.parse_and_bind('tab: complete')
atexit.register(lambda: readline.write_history_file(HISTORY_PATH))
if os.path.exists(HISTORY_PATH):
readline.read_history_file(HISTORY_PATH)
code.interact(banner=banner, local=imports)
def shell():
if (not os.environ.get('DATASTORE_APP_ID', None)
and not os.environ.get('DATASTORE_PROJECT_ID', None)):
raise ValueError('Must set either DATASTORE_APP_ID or DATASTORE_PROJECT_ID'
' environment variable.')
ndb.get_context().set_memcache_policy(False)
ndb.get_context().set_cache_policy(False)
# ndb will set the application ID.
application_id = os.environ['APPLICATION_ID']
id_resolver = datastore_pbs.IdResolver((application_id,))
project_id = id_resolver.resolve_project_id(application_id)
banner = """ndb shell
Python %s
Project: %s
The ndb module is already imported.
""" % (sys.version, project_id)
imports = {
'ndb': ndb,
}
# set up the environment
os.environ['SERVER_SOFTWARE'] = 'Development (ndb_shell)/0.1'
sys.ps1 = '%s> ' % project_id
if readline is not None:
# set up readline
readline.parse_and_bind('tab: complete')
atexit.register(lambda: readline.write_history_file(HISTORY_PATH))
if os.path.exists(HISTORY_PATH):
readline.read_history_file(HISTORY_PATH)
code.interact(banner=banner, local=imports)
def main():
tb = testbed.Testbed()
tb.activate()
tb.init_datastore_v3_stub()
tb.init_memcache_stub()
ctx = ndb.get_context()
ctx.set_cache_policy(False)
ctx.set_memcache_policy(False)
print DbNotes.danotes
note1 = Note(text='blah', when=int(time.time()))
print 'Before:', note1
ent = DbNote(note=note1)
ent.put()
print 'After:', ent.key.get()
print '-' * 20
note2 = Note(text=u'blooh\u1234\U00102345blooh', when=0)
notes = Notes(notes=[note1, note2])
print 'Before:', notes
ent = DbNotes(danotes=notes)
print 'Entity:', ent
print ent._to_pb(set_key=False)
ent.put()
pb = ent._to_pb()
ent2 = DbNotes._from_pb(pb)
print 'After:', ent.key.get()
print '-' * 20
req = GetNotesRequest(on_or_before=42)
class M(ndb.Model):
req = MessageProperty(GetNotesRequest)
m = M(req=req)
print m
print m.put().get()
tb.deactivate()
def main():
tb = testbed.Testbed()
tb.activate()
tb.init_datastore_v3_stub()
tb.init_memcache_stub()
ctx = ndb.get_context()
ctx.set_cache_policy(False)
ctx.set_memcache_policy(False)
print DbNotes.danotes
note1 = Note(text='blah', when=int(time.time()))
print 'Before:', note1
ent = DbNote(note=note1)
ent.put()
print 'After:', ent.key.get()
print '-' * 20
note2 = Note(text=u'blooh\u1234\U00102345blooh', when=0)
notes = Notes(notes=[note1, note2])
print 'Before:', notes
ent = DbNotes(danotes=notes)
print 'Entity:', ent
print ent._to_pb(set_key=False)
ent.put()
pb = ent._to_pb()
ent2 = DbNotes._from_pb(pb)
print 'After:', ent.key.get()
print '-' * 20
req = GetNotesRequest(on_or_before=42)
class M(ndb.Model):
req = MessageProperty(GetNotesRequest)
m = M(req=req)
print m
print m.put().get()
tb.deactivate()
This is a command-line task list manager. From the command line, first setup the necessary environment variables. export DATASTORE_PROJECT_ID=<my-project-id> export DATASTORE_USE_PROJECT_ID_AS_APP_ID=true """ import ndb # [START build_service] # When running outside of App Engine, ndb caching policies should be adjusted. # The local cache is never evicted, so it should be turned off for any long # running processes. In order to use memcache, App Engine Remote API must be # installed. Note that if you are running ndb both inside and outside of # App Engine, your memcache policies *must* match. Otherwise, calling put may # not invalidate your cache and App Engine ndb will get stale results. ndb.get_context().set_cache_policy(False) ndb.get_context().set_memcache_policy(False) # [END build_service] # [START add_entity] # Define the model we will be using for the tasks. class Task(ndb.Model): description = ndb.StringProperty() created = ndb.DateTimeProperty(auto_now_add=True) done = ndb.BooleanProperty(default=False) def add_task(description): """Adds a new task to the Datastore.
def apply_posts(shard=None,
insertion_post_id=None,
lease_seconds=10,
max_tasks=20):
"""Applies a set of pending posts to a shard.
If shard is None then this function will apply mods for whatever is the
first shard it can find in the pull task queue.
insertion_post_id is the post_id that first caused this apply task to be
enqueued. This task will retry until it applies the insertion_post_id
itself or it can confirm that the insertion_post_id has already been
applied. insertion_post_id may be empty if the apply task is not associated
with a particular post (such as cronjobs/cleanup tasks).
"""
# Do not use caching for NDB in this task queue worker.
ctx = ndb.get_context()
ctx.set_cache_policy(lambda x: False)
ctx.set_memcache_policy(lambda x: False)
# Fetch the new Posts to put in sequence.
queue = taskqueue.Queue(config.pending_queue)
# When no shard is specified, process the first tag we find.
task_list = []
if not shard:
task_list.extend(queue.lease_tasks(lease_seconds, 1))
if not task_list:
logging.debug('apply_posts with no specific shard found no tasks')
return
params = task_list[0].extract_params()
shard = params['shard']
logging.debug('apply_posts with no specific shard found shard=%r',
shard)
# Clear the dirty bit on this shard to start the time horizon.
dirty_bit(shard, clear=True)
# Find tasks pending for the current shard.
task_list.extend(
queue.lease_tasks_by_tag(lease_seconds, max_tasks, tag=str(shard)))
receipt_key_list = []
new_topic = None
for task in task_list:
params = task.extract_params()
# Extract the new topic shard associated with this task, if any. The
# last one wins. If all of the found posts have already been applied,
# then topic assignment will be ignored.
new_topic = params.get('new_topic') or new_topic
post_id_list = params.get('post_ids')
if post_id_list is None:
# This may happen on replica shards if it turns out there are no
# unapplied post IDs but an apply task still ran.
post_id_list = []
elif not isinstance(post_id_list, list):
post_id_list = [post_id_list]
for post_id in post_id_list:
receipt_key = ndb.Key(
models.Post._get_kind(), post_id,
models.Receipt._get_kind(), shard)
receipt_key_list.append(receipt_key)
receipt_list = ndb.get_multi(receipt_key_list)
# Some tasks may be in the pull queue that were already put in sequence.
# So ignore these and only apply the new ones.
unapplied_receipts = [
models.Receipt(key=k)
for k, r in zip(receipt_key_list, receipt_list)
if r is None]
unapplied_post_ids = [r.post_id for r in unapplied_receipts]
# Double check if we think there should be work to apply but we didn't find
# any. This will force the apply task to retry immediately if the post task
# was not found. This can happen when the pull queue's consistency is
# behind.
if not unapplied_receipts and insertion_post_id:
receipt_key = ndb.Key(
models.Post._get_kind(), insertion_post_id,
models.Receipt._get_kind(), shard)
receipt = receipt_key.get()
if receipt:
logging.warning(
'No post application to do for shard=%r, but post_id=%r '
'already applied; doing nothing in this task',
shard, insertion_post_id)
new_topic = None
# Do not 'return' here. We need to increment the shard sequence or
# else tasks will not run for this shard in the future because of
# de-duping.
else:
raise base.Error('No post application to do for shard=%r, but'
'post_id=%r has not been applied; will retry' %
(shard, insertion_post_id))
now = datetime.datetime.now()
def txn():
shard_record = models.Shard.get_by_id(shard)
# TODO(bslatkin): Just drop this task entirely if the shard cannot
# be found. Could happen for old shards that were cleaned up.
assert shard_record
# One of the tasks in this batch has a topic assignment. Apply it here.
if new_topic:
logging.debug('Changing topic from %r to %r',
shard_record.current_topic, new_topic)
shard_record.current_topic = new_topic
shard_record.topic_change_time = now
new_sequence_numbers = list(xrange(
shard_record.sequence_number,
shard_record.sequence_number + len(unapplied_receipts)))
shard_record.sequence_number += max(1, len(unapplied_receipts))
# Write post references that point at the newly sequenced posts.
to_put = [shard_record]
for receipt, sequence in zip(unapplied_receipts, new_sequence_numbers):
to_put.append(models.PostReference(
id=sequence,
parent=shard_record.key,
post_id=receipt.post_id))
# Update the receipt entity here; it will be written outside this
# transaction, since these receipts may span multiple entity
# groups.
receipt.sequence = sequence
# Enqueue replica posts transactionally, to make sure everything
# definitely will get copied over to the replica shard.
if shard_record.current_topic:
enqueue_post_task(shard_record.current_topic, unapplied_post_ids)
ndb.put_multi(to_put)
return shard_record, new_sequence_numbers
# Have this only attempt a transaction a single time. If the transaction
# fails the task queue will retry this task within 4 seconds. Because
# apply tasks are always named by the current Shard.sequence_number we
# can be reasonably sure that no other apply task for this shard will be
# running concurrently when this fails.
shard_record, new_sequence_numbers = ndb.transaction(txn, retries=1)
replica_shard = shard_record.current_topic
logging.debug('Applied %d posts for shard=%r, sequence_numbers=%r',
len(unapplied_receipts), shard, new_sequence_numbers)
futures = []
# Save receipts for all the posts.
futures.extend(ndb.put_multi_async(unapplied_receipts))
# Notify all logged in users of the new posts.
futures.append(notify_posts(
shard, unapplied_post_ids, sequence_numbers=new_sequence_numbers))
# Replicate posts to a topic shard.
if replica_shard:
logging.debug('Replicating source shard=%r to replica shard=%r',
shard, replica_shard)
futures.append(enqueue_apply_task(replica_shard))
# Success! Delete the tasks from this queue.
queue.delete_tasks(task_list)
# Always run one more apply task to clean up any posts that came in
# while this transaction was processing.
if dirty_bit(shard, check=True):
futures.append(enqueue_apply_task(shard))
# Wait on all pending futures in case they raise errors.
ndb.Future.wait_all(futures)
# For root shards, add shard cleanup task to check for user presence and
# cause notification of user logouts if the channel API did not detect the
# user closing the connection.
if not shard_record.root_shard:
presence.enqueue_cleanup_task(shard)
This is a command-line task list manager.
From the command line, first setup the necessary environment variables.
export DATASTORE_PROJECT_ID=<my-project-id>
export DATASTORE_USE_PROJECT_ID_AS_APP_ID=true
"""
import ndb
# [START build_service]
# When running outside of App Engine, ndb caching policies should be adjusted.
# The local cache is never evicted, so it should be turned off for any long
# running processes. In order to use memcache, App Engine Remote API must be
# installed. Note that if you are running ndb both inside and outside of
# App Engine, your memcache policies *must* match. Otherwise, calling put may
# not invalidate your cache and App Engine ndb will get stale results.
ndb.get_context().set_cache_policy(False)
ndb.get_context().set_memcache_policy(False)
# [END build_service]
# [START add_entity]
# Define the model we will be using for the tasks.
class Task(ndb.Model):
description = ndb.StringProperty()
created = ndb.DateTimeProperty(auto_now_add=True)
done = ndb.BooleanProperty(default=False)
def add_task(description):
"""Adds a new task to the Datastore.
