Exemple #1
0
    def insert(self, data_list, return_id=False):
        opts = self.query.get_meta()
        unindexed_fields = get_model_indexes(self.query.model)['unindexed']
        unindexed_cols = [
            opts.get_field(name).column for name in unindexed_fields
        ]

        entity_list = []
        for data in data_list:
            properties = {}
            kwds = {'unindexed_properties': unindexed_cols}
            for column, value in data.items():
                # The value will already be a db.Key, but the Entity
                # constructor takes a name or id of the key, and will
                # automatically create a new key if neither is given.
                if column == opts.pk.column:
                    if value is not None:
                        kwds['id'] = value.id()
                        kwds['name'] = value.name()

                # GAE does not store empty lists (and even does not allow
                # passing empty lists to Entity.update) so skip them.
                elif isinstance(value, (tuple, list)) and not len(value):
                    continue

                # Use column names as property names.
                else:
                    properties[column] = value

            entity = Entity(opts.db_table, **kwds)
            entity.update(properties)
            entity_list.append(entity)

        keys = Put(entity_list)
        return keys[0] if isinstance(keys, list) else keys
Exemple #2
0
    def insert(self, data_list, return_id=False):
        opts = self.query.get_meta()
        unindexed_fields = get_model_indexes(self.query.model)['unindexed']
        unindexed_cols = [opts.get_field(name).column
                          for name in unindexed_fields]

        entity_list = []
        for data in data_list:
            properties = {}
            kwds = {'unindexed_properties': unindexed_cols}
            for column, value in data.items():
                # The value will already be a db.Key, but the Entity
                # constructor takes a name or id of the key, and will
                # automatically create a new key if neither is given.
                if column == opts.pk.column:
                    if value is not None:
                        kwds['id'] = value.id()
                        kwds['name'] = value.name()

                # GAE does not store empty lists (and even does not allow
                # passing empty lists to Entity.update) so skip them.
                elif isinstance(value, (tuple, list)) and not len(value):
                    continue

                # Use column names as property names.
                else:
                    properties[column] = value

            entity = Entity(opts.db_table, **kwds)
            entity.update(properties)
            entity_list.append(entity)

        keys = Put(entity_list)
        return keys[0] if isinstance(keys, list) else keys
Exemple #3
0
    def insert(self, data, return_id=False):
        opts = self.query.get_meta()
        unindexed_fields = get_model_indexes(self.query.model)['unindexed']
        kwds = {'unindexed_properties': []}
        properties = {}
        for field, value in data.iteritems():

            # The value will already be a db.Key, but the Entity
            # constructor takes a name or id of the key, and will
            # automatically create a new key if neither is given.
            if field.primary_key:
                if value is not None:
                    kwds['id'] = value.id()
                    kwds['name'] = value.name()

            # GAE does not store empty lists (and even does not allow
            # passing empty lists to Entity.update) so skip them.
            elif isinstance(value, (tuple, list)) and not len(value):
                continue

            # Use column names as property names.
            else:
                properties[field.column] = value

            if field in unindexed_fields:
                kwds['unindexed_properties'].append(field.column)

        entity = Entity(opts.db_table, **kwds)
        entity.update(properties)
        return Put(entity)
Exemple #4
0
    def insert(self, data, return_id=False):
        gae_data = {}
        opts = self.query.get_meta()
        indexes = get_indexes().get(self.query.model, {})
        unindexed_fields = indexes.get('unindexed', ())
        unindexed_cols = [opts.get_field(name).column
                          for name in unindexed_fields]
        kwds = {'unindexed_properties': unindexed_cols}
        for column, value in data.items():
            if column == opts.pk.column:
                if isinstance(value, basestring):
                    kwds['name'] = value
                else:
                    kwds['id'] = value
            elif isinstance(value, (tuple, list)) and not len(value):
                # gae does not store emty lists (and even does not allow passing empty
                # lists to Entity.update) so skip them
                continue
            else:
                gae_data[column] = value

        entity = Entity(self.query.get_meta().db_table, **kwds)
        entity.update(gae_data)
        key = Put(entity)
        return key.id_or_name()
    def insert(self, data, return_id=False):
        gae_data = {}
        opts = self.query.get_meta()
        indexes = get_indexes().get(self.query.model, {})
        unindexed_fields = indexes.get('unindexed', ())
        unindexed_cols = [
            opts.get_field(name).column for name in unindexed_fields
        ]
        kwds = {'unindexed_properties': unindexed_cols}
        for column, value in data.items():
            if column == opts.pk.column:
                if isinstance(value, basestring):
                    kwds['name'] = value
                else:
                    kwds['id'] = value
            elif isinstance(value, (tuple, list)) and not len(value):
                # gae does not store emty lists (and even does not allow passing empty
                # lists to Entity.update) so skip them
                continue
            else:
                gae_data[column] = value

        entity = Entity(self.query.get_meta().db_table, **kwds)
        entity.update(gae_data)
        key = Put(entity)
        return key.id_or_name()
Exemple #6
0
 def run_query(self, query):
     query_pb = query._ToPb()
     encoded_response = yield self._make_request('RunQuery',
                                                 query_pb.Encode())
     results_pb = datastore_pb.QueryResult(encoded_response)
     raise gen.Return(
         [Entity.FromPb(entity) for entity in results_pb.result_list()])
Exemple #7
0
 def run_query(self, query):
     entities = []
     query_pb = query._ToPb()
     encoded_response = yield self._make_request('RunQuery',
                                                 query_pb.Encode())
     results_pb = datastore_pb.QueryResult(encoded_response)
     entities.extend(
         [Entity.FromPb(entity) for entity in results_pb.result_list()])
     while results_pb.has_more_results() and len(
             results_pb.result_list()) >= 100:
         query_pb.set_offset(query_pb.offset() +
                             len(results_pb.result_list()))
         encoded_response = yield self._make_request(
             'RunQuery', query_pb.Encode())
         results_pb = datastore_pb.QueryResult(encoded_response)
         entities.extend(
             [Entity.FromPb(entity) for entity in results_pb.result_list()])
     raise gen.Return(entities)
Exemple #8
0
    def test_cassandra_page_size(self):
        entity_count = self.CASSANDRA_PAGE_SIZE + 1
        for _ in range(entity_count):
            entity = Entity('Greeting', _app=PROJECT_ID)
            yield self.datastore.put(entity)

        query = Query('Greeting', _app=PROJECT_ID)
        results = yield self.datastore.run_query(query)
        self.assertEqual(len(results), entity_count)
        self.assertTrue(True)
Exemple #9
0
  def test_merge_query_with_null(self):
    entity = Entity('Greeting', _app=PROJECT_ID)
    create_time = datetime.datetime.now()
    entity['content'] = None
    entity['create_time'] = create_time
    yield self.datastore.put(entity)

    entity = Entity('Greeting', _app=PROJECT_ID)
    entity['content'] = 'hi'
    entity['create_time'] = create_time
    yield self.datastore.put(entity)

    query = Query('Greeting', {'content =': None, 'create_time =': create_time},
                  _app=PROJECT_ID)
    response = yield self.datastore.run_query(query)
    self.assertEqual(len(response), 1)

    entity = response[0]
    self.assertEqual(entity['content'], None)
    self.assertEqual(entity['create_time'], create_time)
Exemple #10
0
 def test_separator_in_kind(self):
     # The Cloud Datastore API allows these key names, but AppScale forbids them
     # because ':' is used to separate kind names and key names when encoding a
     # path.
     entity = Entity('Invalid:Kind', _app=PROJECT_ID)
     try:
         yield self.datastore.put(entity)
     except BadRequest:
         pass
     else:
         raise Exception('Expected BadRequest. No error was thrown.')
Exemple #11
0
    def test_batch_put_index_entries(self):
        entities = []

        entity = Entity('Greeting', name='duplicate', _app=PROJECT_ID)
        entity['content'] = 'first entry'
        entities.append(entity)

        entity = Entity('Greeting', name='duplicate', _app=PROJECT_ID)
        entity['content'] = 'second entry'
        entities.append(entity)

        yield self.datastore.put_multi(entities)

        # Ensure the last specified mutation is the one that matters.
        query = Query('Greeting', projection=['content'], _app=PROJECT_ID)
        response = yield self.datastore.run_query(query)
        self.assertEqual(len(response), 1)

        entity = response[0]
        self.assertEqual(entity['content'], 'second entry')
Exemple #12
0
    def insert(self, data, return_id=False):
        kwds = {}
        gae_data = {}
        for column, value in data.items():
            if column == self.query.get_meta().pk.column:
                if isinstance(value, basestring):
                    kwds['name'] = value
                else:
                    kwds['id'] = value
            elif isinstance(value, (tuple, list)) and not len(value):
                # gae does not store emty lists (and even does not allow passing empty
                # lists to Entity.update) so skip them
                continue
            else:
                gae_data[column] = value

        entity = Entity(self.query.get_meta().db_table, **kwds)
        entity.update(gae_data)
        key = Put(entity)
        return key.id_or_name()
Exemple #13
0
    def insert(self, data, return_id=False):
        gae_data = {}
        opts = self.query.get_meta()
        unindexed_fields = get_model_indexes(self.query.model)['unindexed']
        unindexed_cols = [opts.get_field(name).column
                          for name in unindexed_fields]
        kwds = {'unindexed_properties': unindexed_cols}
        for column, value in data.items():
            if column == opts.pk.column:
                if isinstance(value, GAEKey):
                    if value.parent_key() and value.parent_key().has_real_key():
                        kwds['parent'] = value.parent_key().real_key()
                    if isinstance(value.id_or_name(), basestring):
                        kwds['name'] = value.id_or_name()
                    elif value.id_or_name() is not None:
                        kwds['id'] = value.id_or_name()
                elif isinstance(value, Key):
                    kwds['parent'] = value.parent()
                    if value.name():
                        kwds['name'] = value.name()
                    elif value.id():
                        kwds['id'] = value.id()
                elif isinstance(value, basestring):
                    kwds['name'] = value
                else:
                    kwds['id'] = value
            elif isinstance(value, (tuple, list)) and not len(value):
                # gae does not store empty lists (and even does not allow passing empty
                # lists to Entity.update) so skip them
                continue
            else:
                gae_data[column] = value

        entity = Entity(opts.db_table, **kwds)
        entity.update(gae_data)
        key = Put(entity)

        if not isinstance(opts.pk, GAEKeyField):
            key = key.id_or_name()
        
        return key
Exemple #14
0
def pb_rec(request):
    if request.method != 'POST':
        return Response('POST required')
    f = request.files.get('file')
    while True:
        try:
            (l,) = unpack('I', f.read(4))
            info(l)
            e = Entity._FromPb(f.read(l))
            info(e)
        except Exception, e:
            info(e)
            break
Exemple #15
0
    def test_cassandra_page_size(self):
        entity_count = self.CASSANDRA_PAGE_SIZE + 1
        batch = []
        for _ in range(entity_count):
            entity = Entity('Greeting', _app=PROJECT_ID)
            batch.append(entity)
            if len(batch) == self.BATCH_SIZE:
                yield self.datastore.put_multi(batch)
                batch = []
        yield self.datastore.put_multi(batch)

        query = Query('Greeting', _app=PROJECT_ID)
        results = yield self.datastore.run_query(query)
        self.assertEqual(len(results), entity_count)
Exemple #16
0
 def perform_insert():
     """
         Inserts result, and any descendents with their ancestor
         value set
     """
     inserted_key = datastore.Put(result)
     if descendents:
         for i, descendent in enumerate(descendents):
             descendents[i] = Entity(
                 descendent.kind(),
                 parent=inserted_key,
                 namespace=inserted_key.namespace(),
                 id=descendent.key().id() or None,
                 name=descendent.key().name() or None)
             descendents[i].update(descendent)
         datastore.Put(descendents)
Exemple #17
0
  def test_separator_in_name(self):
    entity = Entity('Greeting', name='Test:1', _app=PROJECT_ID)
    create_time = datetime.datetime.utcnow()
    entity['color'] = 'red'
    entity['create_time'] = create_time
    yield self.datastore.put(entity)

    query = Query('Greeting', {'color =': 'red', 'create_time =': create_time},
                  _app=PROJECT_ID)
    response = yield self.datastore.run_query(query)

    self.assertEqual(len(response), 1)

    entity = response[0]
    self.assertEqual(entity['color'], 'red')
    self.assertEqual(entity['create_time'], create_time)
Exemple #18
0
    def get(self, key, txid=None):
        request = datastore_pb.GetRequest()
        req_key = request.add_key()
        req_key.MergeFrom(key._ToPb())

        if txid is not None:
            req_tx = request.mutable_transaction()
            req_tx.set_app(self.project_id)
            req_tx.set_handle(txid)

        encoded_response = yield self._make_request('Get', request.Encode())
        get_response = datastore_pb.GetResponse(encoded_response)
        response_entity = get_response.entity(0).entity()
        if not response_entity.has_key():
            return

        raise gen.Return(Entity.FromPb(response_entity))
Exemple #19
0
    def finish(self):

        try:
            if self._keys_only:
                results = [Key._FromPb(e.key()) for e in self._buffer[:self._limit]]
            else:
                results = [Entity._FromPb(e) for e in self._buffer[:self._limit]]
                if self._model_class is not None:
                    from_entity = self._model_class.from_entity
                    results = [from_entity(e) for e in results]
                else:
                    results = [class_for_kind(e.kind()).from_entity(e) for e in results]
        finally:
            del self._buffer[:]

        if self.on_complete:
            results = self.on_complete(results)
        self.set_result(self._value, results)
Exemple #20
0
        def perform_insert(entities):
            results = []
            for primary, descendents in entities:
                new_key = rpc.Put(primary)
                if descendents:
                    for i, descendent in enumerate(descendents):
                        descendents[i] = Entity(
                            descendent.kind(),
                            parent=new_key,
                            namespace=new_key.namespace(),
                            id=descendent.key().id() or None,
                            name=descendent.key().name() or None
                        )
                        descendents[i].update(descendent)

                    rpc.Put(descendents)
                results.append(new_key)
            return results
Exemple #21
0
def increment_counter(datastore, counter_id, retries):
  txid = yield datastore.begin_transaction()
  key = Key.from_path('Counter', counter_id, _app=PROJECT_ID)
  entity = yield datastore.get(key, txid=txid)
  if entity is None:
    entity = Entity('Counter', name=counter_id, _app=PROJECT_ID)

  if 'count' not in entity:
    entity['count'] = 0

  entity['count'] += 1
  yield datastore.put(entity, txid=txid)
  try:
    yield datastore.commit(txid)
  except DatastoreError:
    if retries < 1:
      raise

    yield increment_counter(datastore, counter_id, retries - 1)
Exemple #22
0
def test_separator_in_name(locations):
    datastore = Datastore(locations, PROJECT_ID)

    entity = Entity('Greeting', name='Test:1', _app=PROJECT_ID)
    create_time = datetime.datetime.utcnow()
    entity['color'] = 'red'
    entity['create_time'] = create_time
    yield datastore.put(entity)

    query = Query('Greeting', {
        'color =': 'red',
        'create_time =': create_time
    },
                  _app=PROJECT_ID)
    response = yield datastore.run_query(query)
    if len(response) != 1:
        raise Exception('Expected 1 result. Received: {}'.format(response))

    entity = response[0]
    if entity['color'] != 'red' or entity['create_time'] != create_time:
        raise Exception('Unexpected entity: {}'.format(entity))
Exemple #23
0
def test_merge_query_with_null(locations):
    datastore = Datastore(locations, PROJECT_ID)

    query = Query('Greeting', _app=PROJECT_ID)
    results = yield datastore.run_query(query)
    for entity in results:
        yield datastore.delete([entity.key()])

    entity = Entity('Greeting', _app=PROJECT_ID)
    create_time = datetime.datetime.now()
    entity['content'] = None
    entity['create_time'] = create_time
    yield datastore.put(entity)

    entity = Entity('Greeting', _app=PROJECT_ID)
    entity['content'] = 'hi'
    entity['create_time'] = create_time
    yield datastore.put(entity)

    entity = Entity('Greeting', _app=PROJECT_ID)
    entity['create_time'] = None
    yield datastore.put(entity)

    query = Query('Greeting', {
        'content =': None,
        'create_time =': create_time
    },
                  _app=PROJECT_ID)
    response = yield datastore.run_query(query)
    if len(response) != 1:
        raise Exception('Expected 1 result. Received: {}'.format(response))

    entity = response[0]
    if entity['content'] is not None or entity['create_time'] != create_time:
        raise Exception('Unexpected entity: {}'.format(entity))

    query = Query('Greeting', _app=PROJECT_ID)
    results = yield datastore.run_query(query)
    for entity in results:
        yield datastore.delete([entity.key()])
Exemple #24
0
    def prepare_entity(instance):
        if isinstance(instance.pk, AncestorKey):
            parent = instance.pk._parent_key
        else:
            parent = None

        result = Entity(instance._meta.db_table, parent=parent)

        for field in instance._meta.fields:
            if field.name == "id":
                continue

            value = field.pre_save(instance, True)
            setattr(instance, field.name, value)
            value = field.get_db_prep_save(getattr(instance, field.attname),
                                           connection)
            if isinstance(value, (list, set)):
                value = list(value)
                if not value:
                    value = None

            result[field.column] = value
        return result
Exemple #25
0
    def prep_value_for_database(self, value, index, model, column):
        if value is None:
            return None

        # If this a date or a datetime, or something that supports isoformat, then use that
        if hasattr(value, "isoformat"):
            value = value.isoformat()

        if _is_iterable(value):
            value = list(
                chain(*[self._generate_permutations(v) for v in value]))
        else:
            value = self._generate_permutations(value)

        if not value:
            return None

        value = list(set(value))  # De-duplicate

        entity = Entity(self._generate_kind_name(model, column),
                        name=self.OPERATOR)
        entity[self.INDEXED_COLUMN_NAME] = value
        return [entity]
Exemple #26
0
def process_query_result(result):
    if result.keys_only():
        return [Key._FromPb(e.key()) for e in result.result_list()]
    else:
        return [Entity._FromPb(e) for e in result.result_list()]
Exemple #27
0
        logging.debug("Exception (Delete):%s", exp)
        exception.append(exp)
    if callback:
        callback(rpc)


def get_rpc_handler(rpc):
  try:
      rpc.check_success()
  except apiproxy_errors.ApplicationError, err:
    raise _ToDatastoreError(err)
  resp = rpc.response
  entities = []
  for group in resp.entity_list():
    if group.has_entity():
      entities.append(Entity._FromPb(group.entity()))
    else:
      entities.append(None)

  return entities

def get_callback(rpc, result, exception, callback=None):
    try:
        assert isinstance(rpc.request, datastore_pb.GetRequest), "request should be a GetRequest"
        assert isinstance(rpc.response, datastore_pb.GetResponse), "response should be a GetResponse"

        response = get_rpc_handler(rpc)
        result.append(response)
    except (datastore_errors.Error, apiproxy_errors.Error), exp:
        logging.debug("Exception (Get):%s", exp)
        exception.append(exp)
Exemple #28
0
 def get(self):
     player_entity = Entity('mykind', name = 'e_name')
     player_entity['var'] = 1192
     datastore.Put(player_entity)
     path = os.path.join(os.path.dirname(__file__), 'view/index.html')
     self.response.out.write(template.render(path, {}))