def sum_add(instance, field, aggregator_instance, aggregator_field, process_func=None): """ Shortcut function for adding sum operation. Requires next arguments: * instance - instance of aggregated model; * field - name of field the sum is calculated by; * aggregator_instance - instance of the model the aggregation created for; * aggregator_field - name of the field of aggregator_instance which is AggregatedProperty; * process_func - input value processing function, usually is a lambda function like lambda x: abs(x); """ if process_func is None: process_func = lambda x:x ndb.get_context().clear_cache() if instance.key.id(): old_instance = instance.key.get() else: old_instance = None sum_part = process_func(getattr(instance, field)) if old_instance: if process_func(getattr(old_instance, field)) != sum_part: sum_part -= process_func(getattr(old_instance, field)) else: return #import pdb; pdb.set_trace() setattr(aggregator_instance, aggregator_field, getattr(aggregator_instance, aggregator_field) + sum_part)
def shell(): if (not os.environ.get('DATASTORE_APP_ID', None) and not os.environ.get('DATASTORE_PROJECT_ID', None)): raise ValueError( 'Must set either DATASTORE_APP_ID or DATASTORE_PROJECT_ID' ' environment variable.') ndb.get_context().set_memcache_policy(False) ndb.get_context().set_cache_policy(False) # ndb will set the application ID. application_id = os.environ['APPLICATION_ID'] id_resolver = datastore_pbs.IdResolver((application_id, )) project_id = id_resolver.resolve_project_id(application_id) banner = """ndb shell Python %s Project: %s The ndb module is already imported. """ % (sys.version, project_id) imports = { 'ndb': ndb, } # set up the environment os.environ['SERVER_SOFTWARE'] = 'Development (ndb_shell)/0.1' sys.ps1 = '%s> ' % project_id if readline is not None: # set up readline readline.parse_and_bind('tab: complete') atexit.register(lambda: readline.write_history_file(HISTORY_PATH)) if os.path.exists(HISTORY_PATH): readline.read_history_file(HISTORY_PATH) code.interact(banner=banner, local=imports)
def shell(): if (not os.environ.get('DATASTORE_APP_ID', None) and not os.environ.get('DATASTORE_PROJECT_ID', None)): raise ValueError('Must set either DATASTORE_APP_ID or DATASTORE_PROJECT_ID' ' environment variable.') ndb.get_context().set_memcache_policy(False) ndb.get_context().set_cache_policy(False) # ndb will set the application ID. application_id = os.environ['APPLICATION_ID'] id_resolver = datastore_pbs.IdResolver((application_id,)) project_id = id_resolver.resolve_project_id(application_id) banner = """ndb shell Python %s Project: %s The ndb module is already imported. """ % (sys.version, project_id) imports = { 'ndb': ndb, } # set up the environment os.environ['SERVER_SOFTWARE'] = 'Development (ndb_shell)/0.1' sys.ps1 = '%s> ' % project_id if readline is not None: # set up readline readline.parse_and_bind('tab: complete') atexit.register(lambda: readline.write_history_file(HISTORY_PATH)) if os.path.exists(HISTORY_PATH): readline.read_history_file(HISTORY_PATH) code.interact(banner=banner, local=imports)
def main(): tb = testbed.Testbed() tb.activate() tb.init_datastore_v3_stub() tb.init_memcache_stub() ctx = ndb.get_context() ctx.set_cache_policy(False) ctx.set_memcache_policy(False) print DbNotes.danotes note1 = Note(text='blah', when=int(time.time())) print 'Before:', note1 ent = DbNote(note=note1) ent.put() print 'After:', ent.key.get() print '-' * 20 note2 = Note(text=u'blooh\u1234\U00102345blooh', when=0) notes = Notes(notes=[note1, note2]) print 'Before:', notes ent = DbNotes(danotes=notes) print 'Entity:', ent print ent._to_pb(set_key=False) ent.put() pb = ent._to_pb() ent2 = DbNotes._from_pb(pb) print 'After:', ent.key.get() print '-' * 20 req = GetNotesRequest(on_or_before=42) class M(ndb.Model): req = MessageProperty(GetNotesRequest) m = M(req=req) print m print m.put().get() tb.deactivate()
This is a command-line task list manager. From the command line, first setup the necessary environment variables. export DATASTORE_PROJECT_ID=<my-project-id> export DATASTORE_USE_PROJECT_ID_AS_APP_ID=true """ import ndb # [START build_service] # When running outside of App Engine, ndb caching policies should be adjusted. # The local cache is never evicted, so it should be turned off for any long # running processes. In order to use memcache, App Engine Remote API must be # installed. Note that if you are running ndb both inside and outside of # App Engine, your memcache policies *must* match. Otherwise, calling put may # not invalidate your cache and App Engine ndb will get stale results. ndb.get_context().set_cache_policy(False) ndb.get_context().set_memcache_policy(False) # [END build_service] # [START add_entity] # Define the model we will be using for the tasks. class Task(ndb.Model): description = ndb.StringProperty() created = ndb.DateTimeProperty(auto_now_add=True) done = ndb.BooleanProperty(default=False) def add_task(description): """Adds a new task to the Datastore.
def apply_posts(shard=None, insertion_post_id=None, lease_seconds=10, max_tasks=20): """Applies a set of pending posts to a shard. If shard is None then this function will apply mods for whatever is the first shard it can find in the pull task queue. insertion_post_id is the post_id that first caused this apply task to be enqueued. This task will retry until it applies the insertion_post_id itself or it can confirm that the insertion_post_id has already been applied. insertion_post_id may be empty if the apply task is not associated with a particular post (such as cronjobs/cleanup tasks). """ # Do not use caching for NDB in this task queue worker. ctx = ndb.get_context() ctx.set_cache_policy(lambda x: False) ctx.set_memcache_policy(lambda x: False) # Fetch the new Posts to put in sequence. queue = taskqueue.Queue(config.pending_queue) # When no shard is specified, process the first tag we find. task_list = [] if not shard: task_list.extend(queue.lease_tasks(lease_seconds, 1)) if not task_list: logging.debug('apply_posts with no specific shard found no tasks') return params = task_list[0].extract_params() shard = params['shard'] logging.debug('apply_posts with no specific shard found shard=%r', shard) # Clear the dirty bit on this shard to start the time horizon. dirty_bit(shard, clear=True) # Find tasks pending for the current shard. task_list.extend( queue.lease_tasks_by_tag(lease_seconds, max_tasks, tag=str(shard))) receipt_key_list = [] new_topic = None for task in task_list: params = task.extract_params() # Extract the new topic shard associated with this task, if any. The # last one wins. If all of the found posts have already been applied, # then topic assignment will be ignored. new_topic = params.get('new_topic') or new_topic post_id_list = params.get('post_ids') if post_id_list is None: # This may happen on replica shards if it turns out there are no # unapplied post IDs but an apply task still ran. post_id_list = [] elif not isinstance(post_id_list, list): post_id_list = [post_id_list] for post_id in post_id_list: receipt_key = ndb.Key( models.Post._get_kind(), post_id, models.Receipt._get_kind(), shard) receipt_key_list.append(receipt_key) receipt_list = ndb.get_multi(receipt_key_list) # Some tasks may be in the pull queue that were already put in sequence. # So ignore these and only apply the new ones. unapplied_receipts = [ models.Receipt(key=k) for k, r in zip(receipt_key_list, receipt_list) if r is None] unapplied_post_ids = [r.post_id for r in unapplied_receipts] # Double check if we think there should be work to apply but we didn't find # any. This will force the apply task to retry immediately if the post task # was not found. This can happen when the pull queue's consistency is # behind. if not unapplied_receipts and insertion_post_id: receipt_key = ndb.Key( models.Post._get_kind(), insertion_post_id, models.Receipt._get_kind(), shard) receipt = receipt_key.get() if receipt: logging.warning( 'No post application to do for shard=%r, but post_id=%r ' 'already applied; doing nothing in this task', shard, insertion_post_id) new_topic = None # Do not 'return' here. We need to increment the shard sequence or # else tasks will not run for this shard in the future because of # de-duping. else: raise base.Error('No post application to do for shard=%r, but' 'post_id=%r has not been applied; will retry' % (shard, insertion_post_id)) now = datetime.datetime.now() def txn(): shard_record = models.Shard.get_by_id(shard) # TODO(bslatkin): Just drop this task entirely if the shard cannot # be found. Could happen for old shards that were cleaned up. assert shard_record # One of the tasks in this batch has a topic assignment. Apply it here. if new_topic: logging.debug('Changing topic from %r to %r', shard_record.current_topic, new_topic) shard_record.current_topic = new_topic shard_record.topic_change_time = now new_sequence_numbers = list(xrange( shard_record.sequence_number, shard_record.sequence_number + len(unapplied_receipts))) shard_record.sequence_number += max(1, len(unapplied_receipts)) # Write post references that point at the newly sequenced posts. to_put = [shard_record] for receipt, sequence in zip(unapplied_receipts, new_sequence_numbers): to_put.append(models.PostReference( id=sequence, parent=shard_record.key, post_id=receipt.post_id)) # Update the receipt entity here; it will be written outside this # transaction, since these receipts may span multiple entity # groups. receipt.sequence = sequence # Enqueue replica posts transactionally, to make sure everything # definitely will get copied over to the replica shard. if shard_record.current_topic: enqueue_post_task(shard_record.current_topic, unapplied_post_ids) ndb.put_multi(to_put) return shard_record, new_sequence_numbers # Have this only attempt a transaction a single time. If the transaction # fails the task queue will retry this task within 4 seconds. Because # apply tasks are always named by the current Shard.sequence_number we # can be reasonably sure that no other apply task for this shard will be # running concurrently when this fails. shard_record, new_sequence_numbers = ndb.transaction(txn, retries=1) replica_shard = shard_record.current_topic logging.debug('Applied %d posts for shard=%r, sequence_numbers=%r', len(unapplied_receipts), shard, new_sequence_numbers) futures = [] # Save receipts for all the posts. futures.extend(ndb.put_multi_async(unapplied_receipts)) # Notify all logged in users of the new posts. futures.append(notify_posts( shard, unapplied_post_ids, sequence_numbers=new_sequence_numbers)) # Replicate posts to a topic shard. if replica_shard: logging.debug('Replicating source shard=%r to replica shard=%r', shard, replica_shard) futures.append(enqueue_apply_task(replica_shard)) # Success! Delete the tasks from this queue. queue.delete_tasks(task_list) # Always run one more apply task to clean up any posts that came in # while this transaction was processing. if dirty_bit(shard, check=True): futures.append(enqueue_apply_task(shard)) # Wait on all pending futures in case they raise errors. ndb.Future.wait_all(futures) # For root shards, add shard cleanup task to check for user presence and # cause notification of user logouts if the channel API did not detect the # user closing the connection. if not shard_record.root_shard: presence.enqueue_cleanup_task(shard)