Esempio n. 1
0
 def get_result(self):
     if self.cache_result is not None:
         return self.cache_result
     if len(self.exception) >= 1:
         raise self.exception[0]
     else:
         return [db.class_for_kind(e.kind()).from_entity(e) if isinstance(e, Entity) else e for e in self.results]
Esempio n. 2
0
def clear_datastore(appid=('a', ''),
                    host=('h', ''),
                    path=('p', ''),
                    kinds=('k', ''),
                    clear_memcache=('c', False),
                    secure=True):
    """Clear all the data on GAE environment using remote_api.
  """
    if not appid:
        appid = get_appid()
    if not host:
        host = "%s.appspot.com" % appid
    if not path:
        path = '/remote_api'
    if not kinds:
        models = None
    else:
        models_dict = get_all_models_as_dict()
        models = []
        for kind in kinds.split(','):
            models.append(db.class_for_kind(kind))

    remote_api_stub.ConfigureRemoteApi(appid,
                                       path,
                                       auth_func,
                                       host,
                                       secure=secure,
                                       save_cookies=True)
    remote_api_stub.MaybeInvokeAuthentication()
    delete_all_entities(models)
    if clear_memcache:
        from google.appengine.api import memcache
        memcache.flush_all()
        sys.stderr.write("Flushed memcache.\n")
Esempio n. 3
0
 def get_result(self):
     if self.cache_result is not None:
         return self.cache_result
     if len(self.exception) >= 1:
         raise self.exception[0]
     else:
         return [db.class_for_kind(e.kind()).from_entity(e) if isinstance(e, Entity) else e for e in self.results]
Esempio n. 4
0
	def post(self):
		console.log(self.body)
		console.log(self.client)
		if not self.client is None:
			if 'updates' in self.body:
				keys = []
				for key in self.body['updates']:
					try:
						keys.append(db.Key(key))
					except:
						pass
				entities = Entity.get(keys)
				for entity in entities:
					entity.execute(self.body['updates'][str(entity.key())])
				db.put(entities)
				for entity in Entity.gql('updated > :updated', updated = self.client.updated):
					self.data(entity)
			if 'creates' in self.body:
				entities = []
				for name in self.body['creates']:
					kind = db.class_for_kind(name)
					for data in self.body['create'][name]:
						entity = kind(data = data)
						entities.append(entity)
				db.put(entities)
				for entity in entities:
					self.data(entity)
			self.client.updated = datetime.utcnow()
			self.client.put()
			self.data(self.client)
		return self.out()
		
		'''
Esempio n. 5
0
def clear_datastore(appid=('a', ''), host=('h', ''), path=('p', ''),
                    kinds=('k', ''), clear_memcache=('c', False), secure=True):
  """Clear all the data on GAE environment using remote_api.
  """
  if not appid:
    appid = get_appid()
  if not host:
    host = "%s.appspot.com" % appid
  if not path:
    path = DEFAULT_REMOTE_API_PATH
  if not kinds:
    models = None
  else:
    models_dict = get_all_models_as_dict()
    models = []
    for kind in kinds.split(','):
      models.append(db.class_for_kind(kind))
      
  remote_api_stub.ConfigureRemoteApi(None, path, auth_func,
                                     host, secure=secure, save_cookies=True)
  remote_api_stub.MaybeInvokeAuthentication()
  delete_all_entities(models)
  if clear_memcache:
    from google.appengine.api import memcache
    memcache.flush_all()
    sys.stderr.write("Flushed memcache.\n")
Esempio n. 6
0
 def getQuery(self, context, obj):
     """ Query for all entities before a given datetime. """
     model = context['model']
     dateattr = context['dateattr']
     before = context['before']
     modelClass = db.class_for_kind(model)
     query = modelClass.all(keys_only=True, namespace='').filter('%s <' % dateattr, before)
     return query
Esempio n. 7
0
 def getQuery(self, context, obj):
     """ Query for all entities before a given datetime. """
     model = context['model']
     dateattr = context['dateattr']
     before = context['before']
     modelClass = db.class_for_kind(model)
     query = modelClass.all(keys_only=True,
                            namespace='').filter('%s <' % dateattr, before)
     return query
Esempio n. 8
0
def execute_pre_save_hooks(kind, pb_key, entity):
  put_type_id = put_type.UNKNOWN
  func_list = pre_save_hooks.get(kind, None)
  if func_list is not None:
    key = db.Key._FromPb(pb_key)
    if not key.id_or_name():
      path = key.to_path()[:-1]
      path.append(1)
      model_key = db.Key.from_path(*path)
      ids = datastore.AllocateIds(model_key, 1)
      path = path[:-1]
      path.append(ids[0])
      new_key = db.Key.from_path(*path)
      pb_key = new_key._ToPb()
      entity.key().CopyFrom(pb_key)
      group = entity.mutable_entity_group()
      root = entity.key().path().element(0)
      group.add_element().CopyFrom(root)
      e = datastore.Entity._FromPb(entity)
      instance = db.class_for_kind(kind).from_entity(e)
      put_type_id = put_type.NEWLY_CREATED
    else:
      entity.key_.CopyFrom(pb_key)
      e = datastore.Entity._FromPb(entity)
      instance = db.class_for_kind(kind).from_entity(e)
      created = get_created_datetime(instance)
      updated = get_updated_datetime(instance)
      if created:
        threshold = datetime.timedelta(0,0,1000)
        if updated:
          if abs(created - updated) < threshold:
            put_type_id = put_type.MAYBE_NEWLY_CREATED
          else:
            put_type_id = put_type.MAYBE_UPDATED
        else:
          if (datetime.datetime.now() - created) < threshold:
            put_type_id = put_type.MAYBE_NEWLY_CREATED
          else:
            put_type_id = put_type.MAYBE_UPDATED
    for func in func_list:
      func(instance, put_type_id)
Esempio n. 9
0
def execute_pre_save_hooks(kind, pb_key, entity):
    put_type_id = put_type.UNKNOWN
    func_list = pre_save_hooks.get(kind, None)
    if func_list is not None:
        key = db.Key._FromPb(pb_key)
        if not key.id_or_name():
            path = key.to_path()[:-1]
            path.append(1)
            model_key = db.Key.from_path(*path)
            ids = datastore.AllocateIds(model_key, 1)
            path = path[:-1]
            path.append(ids[0])
            new_key = db.Key.from_path(*path)
            pb_key = new_key._ToPb()
            entity.key().CopyFrom(pb_key)
            group = entity.mutable_entity_group()
            root = entity.key().path().element(0)
            group.add_element().CopyFrom(root)
            e = datastore.Entity._FromPb(entity)
            instance = db.class_for_kind(kind).from_entity(e)
            put_type_id = put_type.NEWLY_CREATED
        else:
            entity.key_.CopyFrom(pb_key)
            e = datastore.Entity._FromPb(entity)
            instance = db.class_for_kind(kind).from_entity(e)
            created = get_created_datetime(instance)
            updated = get_updated_datetime(instance)
            if created:
                threshold = datetime.timedelta(0, 0, 1000)
                if updated:
                    if abs(created - updated) < threshold:
                        put_type_id = put_type.MAYBE_NEWLY_CREATED
                    else:
                        put_type_id = put_type.MAYBE_UPDATED
                else:
                    if (datetime.datetime.now() - created) < threshold:
                        put_type_id = put_type.MAYBE_NEWLY_CREATED
                    else:
                        put_type_id = put_type.MAYBE_UPDATED
        for func in func_list:
            func(instance, put_type_id)
Esempio n. 10
0
def dump(obj,):
    key_name = None
    try:
        kind = obj.kind()
        key_name = obj.name()
    except AttributeError:
        pass
    if key_name is None:
        raise TypeError("db.Key instance with a name is required")
    cls = db.class_for_kind(kind)
    try:
        return cls.parse_key_name(key_name)
    except AttributeError:
        return key_name
def encode_xdb_key(key, encoder=None):
    """
    Convert the `db.Key` to it's entity and encode it.
    """
    gae_objects = getGAEObjects(encoder.context.extra)

    klass = db.class_for_kind(key.kind())

    try:
        referenced_object = gae_objects.get(klass, key)
    except KeyError:
        referenced_object = db.get(key)
        gae_objects.set(klass, key, referenced_object)

    if not referenced_object:
        encoder.writeElement(None)
    else:
        encoder.writeObject(referenced_object)
Esempio n. 12
0
def encode_xdb_key(key, encoder=None):
    """
    Convert the `db.Key` to it's entity and encode it.
    """
    gae_objects = getGAEObjects(encoder.context.extra)

    klass = db.class_for_kind(key.kind())

    try:
        referenced_object = gae_objects.get(klass, key)
    except KeyError:
        referenced_object = db.get(key)
        gae_objects.set(klass, key, referenced_object)

    if not referenced_object:
        encoder.writeElement(None)
    else:
        encoder.writeObject(referenced_object)
Esempio n. 13
0
def execute_hooks(kind, key, entity):
  put_type_id = put_type.UNKOWN
  func_list = post_save_hooks.get(kind, None)
  if func_list is not None:
    last_path = entity.key().path().element_list()[-1]
    has_name = last_path.has_name()
    if last_path.has_id():
      if last_path.id() == 0:
        put_type_id = put_type.NEWLY_CREATED
      else:
        put_type_id = put_type.UPDATED
    entity.key_.CopyFrom(key)
    e = datastore.Entity._FromPb(entity)
    instance = db.class_for_kind(kind).from_entity(e)
    if has_name:
      created = get_created_datetime(instance)
      updated = get_updated_datetime(instance)
      if created:
        import datetime
        threshold = datetime.timedelta(0,0,1000)
        if updated:
          if abs(created - updated) < threshold:
            put_type_id = put_type.MAYBE_NEWLY_CREATED
          else:
            put_type_id = put_type.MAYBE_UPDATED
        else:
          if (datetime.datetime.now() - created) < threshold:
            put_type_id = put_type.MAYBE_NEWLY_CREATED
          else:
            put_type_id = put_type.MAYBE_UPDATED
    if datastore._CurrentTransactionKey():
      # This operation is inside the transaction. So, we reserve the
      # func_list and parameters for later execution.
      tmp_list = getattr(local, '_reserved_hooks', None)
      if tmp_list is None:
        tmp_list = []
      tmp_list.append((func_list, instance, put_type_id))
      local._reserved_hooks = tmp_list
    else:
      for func in func_list:
        func(instance, put_type_id)
Esempio n. 14
0
def execute_post_save_hooks(kind, key, entity):
    put_type_id = put_type.UNKNOWN
    func_list = post_save_hooks.get(kind, None)
    if func_list is not None:
        last_path = entity.key().path().element_list()[-1]
        has_name = last_path.has_name()
        if last_path.has_id():
            if last_path.id() == 0:
                key_auto_generated = True
                put_type_id = put_type.NEWLY_CREATED
            else:
                key_auto_generated = False
        entity.key_.CopyFrom(key)
        e = datastore.Entity._FromPb(entity)
        instance = db.class_for_kind(kind).from_entity(e)
        if has_name or not key_auto_generated:
            created = get_created_datetime(instance)
            updated = get_updated_datetime(instance)
            if created:
                threshold = datetime.timedelta(0, 0, 1500)
                if updated:
                    if abs(created - updated) < threshold:
                        put_type_id = put_type.MAYBE_NEWLY_CREATED
                    else:
                        put_type_id = put_type.MAYBE_UPDATED
                else:
                    if (datetime.datetime.now() - created) < threshold:
                        put_type_id = put_type.MAYBE_NEWLY_CREATED
                    else:
                        put_type_id = put_type.MAYBE_UPDATED
        if datastore._CurrentTransactionKey():
            # This operation is inside the transaction. So, we reserve the
            # func_list and parameters for later execution.
            tmp_list = getattr(local, '_reserved_hooks', [])
            tmp_list.append((func_list, instance, put_type_id))
            local._reserved_hooks = tmp_list
        else:
            for func in func_list:
                func(instance, put_type_id)
Esempio n. 15
0
 def __init__(self, obj): 
     self.val = db.model_to_protobuf(obj).Encode()
     self.model_class = db.class_for_kind(obj.kind())
Esempio n. 16
0
 def __init__(self, db_model_name):
     self._model = db.class_for_kind(db_model_name)
Esempio n. 17
0
def benchmark_serialization(response, model_instance):
    # How data gets from a db.Model subclass to bytes:
    # 1. db.Model is converted to a datastore.Entity
    # 2. datastore.Entity is converted to a protocol buffer object: EntityProto
    # 3. EntityProto is serialized
    start = time.time()
    for _ in xrange(SERIALIZATION_ITERATIONS):
        entity = model_instance._populate_entity(datastore.Entity)
        entity_proto = entity.ToPb()
        serialized = entity_proto.SerializeToString()
    end = time.time()
    output(
        response, 'serialized from model %d times in %f s' %
        (SERIALIZATION_ITERATIONS, end - start))

    start = time.time()
    for _ in xrange(SERIALIZATION_ITERATIONS):
        entity_proto = entity.ToPb()
        serialized = entity_proto.SerializeToString()
    end = time.time()
    output(
        response, 'serialized from datastore.Entity %d times in %f s' %
        (SERIALIZATION_ITERATIONS, end - start))

    start = time.time()
    for _ in xrange(SERIALIZATION_ITERATIONS):
        serialized = entity_proto.SerializeToString()
    end = time.time()
    output(
        response, 'serialized from entity_pb.EntityProto %d times in %f s' %
        (SERIALIZATION_ITERATIONS, end - start))

    start = time.time()
    for _ in xrange(SERIALIZATION_ITERATIONS):
        entity_proto = entity_pb.EntityProto(serialized)
        entity = datastore.Entity.FromPb(entity_proto)
        # from model_from_protobuf
        deserialized = db.class_for_kind(entity.kind()).from_entity(entity)
    end = time.time()
    output(
        response, 'deserialized to model %d times in %f s' %
        (SERIALIZATION_ITERATIONS, end - start))

    start = time.time()
    for _ in xrange(SERIALIZATION_ITERATIONS):
        entity_proto = entity_pb.EntityProto(serialized)
        entity = datastore.Entity.FromPb(entity_proto)
    end = time.time()
    output(
        response, 'deserialized to datastore.Entity %d times in %f s' %
        (SERIALIZATION_ITERATIONS, end - start))

    start = time.time()
    for _ in xrange(SERIALIZATION_ITERATIONS):
        entity_proto = entity_pb.EntityProto(serialized)
    end = time.time()
    output(
        response, 'deserialized to entity_pb.EntityProto %d times in %f s' %
        (SERIALIZATION_ITERATIONS, end - start))

    start = time.time()
    for _ in xrange(SERIALIZATION_ITERATIONS):
        entity_proto.Clear()
        entity_proto.MergeFromString(serialized)
    end = time.time()
    output(
        response,
        'deserialized to entity_pb.EntityProto with reuse %d times in %f s' %
        (SERIALIZATION_ITERATIONS, end - start))
    output(response, '  (entity has %d keys)' % len(entity))

    # model / LazyEntity property access tests
    response.write('\n### model / LazyEntity property access times\n')
    total_length = 0
    start = time.time()
    for _ in xrange(SERIALIZATION_ITERATIONS):
        entity_proto = entity_pb.EntityProto(serialized)
        entity = datastore.Entity.FromPb(entity_proto)
        # from model_from_protobuf
        deserialized = db.class_for_kind(entity.kind()).from_entity(entity)
        total_length += len(deserialized.prop_a)
    end = time.time()
    output(
        response,
        'model deserialized and accessed one property %d times in %f s' %
        (SERIALIZATION_ITERATIONS, end - start))

    total_length = 0
    start = time.time()
    for _ in xrange(SERIALIZATION_ITERATIONS):
        entity_proto = entity_pb.EntityProto(serialized)
        entity = datastore.Entity.FromPb(entity_proto)
        # from model_from_protobuf
        deserialized = db.class_for_kind(entity.kind()).from_entity(entity)
        total_length += len(deserialized.prop_a)
        total_length += len(deserialized.prop_b)
        total_length += len(deserialized.prop_c)
        total_length += len(deserialized.prop_d)
        total_length += len(deserialized.prop_e)
    end = time.time()
    output(
        response,
        'model deserialized and accessed five properties %d times in %f s' %
        (SERIALIZATION_ITERATIONS, end - start))

    total_length = 0
    start = time.time()
    for _ in xrange(SERIALIZATION_ITERATIONS):
        entity_proto = entity_pb.EntityProto(serialized)
        deserialized = datastore_lazy.LazyEntity(entity_proto)
        total_length += len(deserialized.prop_a)
    end = time.time()
    output(
        response,
        'LazyEntity deserialized and accessed one property %d times in %f s' %
        (SERIALIZATION_ITERATIONS, end - start))

    total_length = 0
    start = time.time()
    for _ in xrange(SERIALIZATION_ITERATIONS):
        entity_proto = entity_pb.EntityProto(serialized)
        deserialized = datastore_lazy.LazyEntity(entity_proto)
        total_length += len(deserialized.prop_a)
        total_length += len(deserialized.prop_b)
        total_length += len(deserialized.prop_c)
        total_length += len(deserialized.prop_d)
        total_length += len(deserialized.prop_e)
    end = time.time()
    output(
        response,
        'LazyEntity deserialized and accessed five properties %d times in %f s'
        % (SERIALIZATION_ITERATIONS, end - start))

    response.write('\n### protocol buffer / pure python access times\n')
    total = 0
    start = time.time()
    for _ in xrange(SERIALIZATION_ITERATIONS):
        total += entity_proto.property_size()
    end = time.time()
    output(
        response, 'protocol buffer entity_proto.property_size access in %f s' %
        (end - start))

    total = 0
    obj = PythonListHolder()
    start = time.time()
    for _ in xrange(SERIALIZATION_ITERATIONS):
        total += obj.property_size()
    end = time.time()
    output(response,
           'PythonListHolder.property_size access in %f s' % (end - start))
Esempio n. 18
0
 def processItem(self, item):
     self.processItemAsEntity(item)
     logging.info("processing %s %s" % (item.kind(), item.key()))
     modelClass = db.class_for_kind(item.kind()).from_entity(item)
     self.processItemAsClass(modelClass)
     modelClass.put()
Esempio n. 19
0
 def _model(self):
   if isinstance(self.__model, basestring):
     self.__model = db.class_for_kind(self.__model)
   return self.__model
Esempio n. 20
0
def create(model):
    _class = db.class_for_kind(model)
    return _class()
Esempio n. 21
0
def makeProtoBufObj(obj):
  val = db.model_to_protobuf(obj).Encode()
  model_class =  db.class_for_kind(obj.kind())
  return ProtoBufObj(val, model_class) 
Esempio n. 22
0
 def __init__(self, db_model_name):
     self._model = db.class_for_kind(db_model_name)
Esempio n. 23
0
 def __init__(self, kind, is_Thing = False):
     object.__setattr__(self, '_model' , db.class_for_kind(kind))
     object.__setattr__(self, '_is_Thing', is_Thing)
     object.__setattr__(self, '_obj', None)
     object.__setattr__(self, '_single_entry', False)