def insert(self, data_list, return_id=False):
opts = self.query.get_meta()
unindexed_fields = get_model_indexes(self.query.model)['unindexed']
unindexed_cols = [
opts.get_field(name).column for name in unindexed_fields
]
entity_list = []
for data in data_list:
properties = {}
kwds = {'unindexed_properties': unindexed_cols}
for column, value in data.items():
# The value will already be a db.Key, but the Entity
# constructor takes a name or id of the key, and will
# automatically create a new key if neither is given.
if column == opts.pk.column:
if value is not None:
kwds['id'] = value.id()
kwds['name'] = value.name()
# GAE does not store empty lists (and even does not allow
# passing empty lists to Entity.update) so skip them.
elif isinstance(value, (tuple, list)) and not len(value):
continue
# Use column names as property names.
else:
properties[column] = value
entity = Entity(opts.db_table, **kwds)
entity.update(properties)
entity_list.append(entity)
keys = Put(entity_list)
return keys[0] if isinstance(keys, list) else keys
def insert(self, data_list, return_id=False):
opts = self.query.get_meta()
unindexed_fields = get_model_indexes(self.query.model)['unindexed']
unindexed_cols = [opts.get_field(name).column
for name in unindexed_fields]
entity_list = []
for data in data_list:
properties = {}
kwds = {'unindexed_properties': unindexed_cols}
for column, value in data.items():
# The value will already be a db.Key, but the Entity
# constructor takes a name or id of the key, and will
# automatically create a new key if neither is given.
if column == opts.pk.column:
if value is not None:
kwds['id'] = value.id()
kwds['name'] = value.name()
# GAE does not store empty lists (and even does not allow
# passing empty lists to Entity.update) so skip them.
elif isinstance(value, (tuple, list)) and not len(value):
continue
# Use column names as property names.
else:
properties[column] = value
entity = Entity(opts.db_table, **kwds)
entity.update(properties)
entity_list.append(entity)
keys = Put(entity_list)
return keys[0] if isinstance(keys, list) else keys
def insert(self, data, return_id=False):
opts = self.query.get_meta()
unindexed_fields = get_model_indexes(self.query.model)['unindexed']
kwds = {'unindexed_properties': []}
properties = {}
for field, value in data.iteritems():
# The value will already be a db.Key, but the Entity
# constructor takes a name or id of the key, and will
# automatically create a new key if neither is given.
if field.primary_key:
if value is not None:
kwds['id'] = value.id()
kwds['name'] = value.name()
# GAE does not store empty lists (and even does not allow
# passing empty lists to Entity.update) so skip them.
elif isinstance(value, (tuple, list)) and not len(value):
continue
# Use column names as property names.
else:
properties[field.column] = value
if field in unindexed_fields:
kwds['unindexed_properties'].append(field.column)
entity = Entity(opts.db_table, **kwds)
entity.update(properties)
return Put(entity)
def insert(self, data, return_id=False):
gae_data = {}
opts = self.query.get_meta()
indexes = get_indexes().get(self.query.model, {})
unindexed_fields = indexes.get('unindexed', ())
unindexed_cols = [opts.get_field(name).column
for name in unindexed_fields]
kwds = {'unindexed_properties': unindexed_cols}
for column, value in data.items():
if column == opts.pk.column:
if isinstance(value, basestring):
kwds['name'] = value
else:
kwds['id'] = value
elif isinstance(value, (tuple, list)) and not len(value):
# gae does not store emty lists (and even does not allow passing empty
# lists to Entity.update) so skip them
continue
else:
gae_data[column] = value
entity = Entity(self.query.get_meta().db_table, **kwds)
entity.update(gae_data)
key = Put(entity)
return key.id_or_name()
def insert(self, data, return_id=False):
gae_data = {}
opts = self.query.get_meta()
indexes = get_indexes().get(self.query.model, {})
unindexed_fields = indexes.get('unindexed', ())
unindexed_cols = [
opts.get_field(name).column for name in unindexed_fields
]
kwds = {'unindexed_properties': unindexed_cols}
for column, value in data.items():
if column == opts.pk.column:
if isinstance(value, basestring):
kwds['name'] = value
else:
kwds['id'] = value
elif isinstance(value, (tuple, list)) and not len(value):
# gae does not store emty lists (and even does not allow passing empty
# lists to Entity.update) so skip them
continue
else:
gae_data[column] = value
entity = Entity(self.query.get_meta().db_table, **kwds)
entity.update(gae_data)
key = Put(entity)
return key.id_or_name()
def run_query(self, query):
query_pb = query._ToPb()
encoded_response = yield self._make_request('RunQuery',
query_pb.Encode())
results_pb = datastore_pb.QueryResult(encoded_response)
raise gen.Return(
[Entity.FromPb(entity) for entity in results_pb.result_list()])
def run_query(self, query):
entities = []
query_pb = query._ToPb()
encoded_response = yield self._make_request('RunQuery',
query_pb.Encode())
results_pb = datastore_pb.QueryResult(encoded_response)
entities.extend(
[Entity.FromPb(entity) for entity in results_pb.result_list()])
while results_pb.has_more_results() and len(
results_pb.result_list()) >= 100:
query_pb.set_offset(query_pb.offset() +
len(results_pb.result_list()))
encoded_response = yield self._make_request(
'RunQuery', query_pb.Encode())
results_pb = datastore_pb.QueryResult(encoded_response)
entities.extend(
[Entity.FromPb(entity) for entity in results_pb.result_list()])
raise gen.Return(entities)
def test_cassandra_page_size(self):
entity_count = self.CASSANDRA_PAGE_SIZE + 1
for _ in range(entity_count):
entity = Entity('Greeting', _app=PROJECT_ID)
yield self.datastore.put(entity)
query = Query('Greeting', _app=PROJECT_ID)
results = yield self.datastore.run_query(query)
self.assertEqual(len(results), entity_count)
self.assertTrue(True)
def test_merge_query_with_null(self):
entity = Entity('Greeting', _app=PROJECT_ID)
create_time = datetime.datetime.now()
entity['content'] = None
entity['create_time'] = create_time
yield self.datastore.put(entity)
entity = Entity('Greeting', _app=PROJECT_ID)
entity['content'] = 'hi'
entity['create_time'] = create_time
yield self.datastore.put(entity)
query = Query('Greeting', {'content =': None, 'create_time =': create_time},
_app=PROJECT_ID)
response = yield self.datastore.run_query(query)
self.assertEqual(len(response), 1)
entity = response[0]
self.assertEqual(entity['content'], None)
self.assertEqual(entity['create_time'], create_time)
def test_separator_in_kind(self):
# The Cloud Datastore API allows these key names, but AppScale forbids them
# because ':' is used to separate kind names and key names when encoding a
# path.
entity = Entity('Invalid:Kind', _app=PROJECT_ID)
try:
yield self.datastore.put(entity)
except BadRequest:
pass
else:
raise Exception('Expected BadRequest. No error was thrown.')
def test_batch_put_index_entries(self):
entities = []
entity = Entity('Greeting', name='duplicate', _app=PROJECT_ID)
entity['content'] = 'first entry'
entities.append(entity)
entity = Entity('Greeting', name='duplicate', _app=PROJECT_ID)
entity['content'] = 'second entry'
entities.append(entity)
yield self.datastore.put_multi(entities)
# Ensure the last specified mutation is the one that matters.
query = Query('Greeting', projection=['content'], _app=PROJECT_ID)
response = yield self.datastore.run_query(query)
self.assertEqual(len(response), 1)
entity = response[0]
self.assertEqual(entity['content'], 'second entry')
def insert(self, data, return_id=False):
kwds = {}
gae_data = {}
for column, value in data.items():
if column == self.query.get_meta().pk.column:
if isinstance(value, basestring):
kwds['name'] = value
else:
kwds['id'] = value
elif isinstance(value, (tuple, list)) and not len(value):
# gae does not store emty lists (and even does not allow passing empty
# lists to Entity.update) so skip them
continue
else:
gae_data[column] = value
entity = Entity(self.query.get_meta().db_table, **kwds)
entity.update(gae_data)
key = Put(entity)
return key.id_or_name()
def insert(self, data, return_id=False):
gae_data = {}
opts = self.query.get_meta()
unindexed_fields = get_model_indexes(self.query.model)['unindexed']
unindexed_cols = [opts.get_field(name).column
for name in unindexed_fields]
kwds = {'unindexed_properties': unindexed_cols}
for column, value in data.items():
if column == opts.pk.column:
if isinstance(value, GAEKey):
if value.parent_key() and value.parent_key().has_real_key():
kwds['parent'] = value.parent_key().real_key()
if isinstance(value.id_or_name(), basestring):
kwds['name'] = value.id_or_name()
elif value.id_or_name() is not None:
kwds['id'] = value.id_or_name()
elif isinstance(value, Key):
kwds['parent'] = value.parent()
if value.name():
kwds['name'] = value.name()
elif value.id():
kwds['id'] = value.id()
elif isinstance(value, basestring):
kwds['name'] = value
else:
kwds['id'] = value
elif isinstance(value, (tuple, list)) and not len(value):
# gae does not store empty lists (and even does not allow passing empty
# lists to Entity.update) so skip them
continue
else:
gae_data[column] = value
entity = Entity(opts.db_table, **kwds)
entity.update(gae_data)
key = Put(entity)
if not isinstance(opts.pk, GAEKeyField):
key = key.id_or_name()
return key
def pb_rec(request):
if request.method != 'POST':
return Response('POST required')
f = request.files.get('file')
while True:
try:
(l,) = unpack('I', f.read(4))
info(l)
e = Entity._FromPb(f.read(l))
info(e)
except Exception, e:
info(e)
break
def test_cassandra_page_size(self):
entity_count = self.CASSANDRA_PAGE_SIZE + 1
batch = []
for _ in range(entity_count):
entity = Entity('Greeting', _app=PROJECT_ID)
batch.append(entity)
if len(batch) == self.BATCH_SIZE:
yield self.datastore.put_multi(batch)
batch = []
yield self.datastore.put_multi(batch)
query = Query('Greeting', _app=PROJECT_ID)
results = yield self.datastore.run_query(query)
self.assertEqual(len(results), entity_count)
def perform_insert():
"""
Inserts result, and any descendents with their ancestor
value set
"""
inserted_key = datastore.Put(result)
if descendents:
for i, descendent in enumerate(descendents):
descendents[i] = Entity(
descendent.kind(),
parent=inserted_key,
namespace=inserted_key.namespace(),
id=descendent.key().id() or None,
name=descendent.key().name() or None)
descendents[i].update(descendent)
datastore.Put(descendents)
def test_separator_in_name(self):
entity = Entity('Greeting', name='Test:1', _app=PROJECT_ID)
create_time = datetime.datetime.utcnow()
entity['color'] = 'red'
entity['create_time'] = create_time
yield self.datastore.put(entity)
query = Query('Greeting', {'color =': 'red', 'create_time =': create_time},
_app=PROJECT_ID)
response = yield self.datastore.run_query(query)
self.assertEqual(len(response), 1)
entity = response[0]
self.assertEqual(entity['color'], 'red')
self.assertEqual(entity['create_time'], create_time)
def get(self, key, txid=None):
request = datastore_pb.GetRequest()
req_key = request.add_key()
req_key.MergeFrom(key._ToPb())
if txid is not None:
req_tx = request.mutable_transaction()
req_tx.set_app(self.project_id)
req_tx.set_handle(txid)
encoded_response = yield self._make_request('Get', request.Encode())
get_response = datastore_pb.GetResponse(encoded_response)
response_entity = get_response.entity(0).entity()
if not response_entity.has_key():
return
raise gen.Return(Entity.FromPb(response_entity))
def finish(self):
try:
if self._keys_only:
results = [Key._FromPb(e.key()) for e in self._buffer[:self._limit]]
else:
results = [Entity._FromPb(e) for e in self._buffer[:self._limit]]
if self._model_class is not None:
from_entity = self._model_class.from_entity
results = [from_entity(e) for e in results]
else:
results = [class_for_kind(e.kind()).from_entity(e) for e in results]
finally:
del self._buffer[:]
if self.on_complete:
results = self.on_complete(results)
self.set_result(self._value, results)
def perform_insert(entities):
results = []
for primary, descendents in entities:
new_key = rpc.Put(primary)
if descendents:
for i, descendent in enumerate(descendents):
descendents[i] = Entity(
descendent.kind(),
parent=new_key,
namespace=new_key.namespace(),
id=descendent.key().id() or None,
name=descendent.key().name() or None
)
descendents[i].update(descendent)
rpc.Put(descendents)
results.append(new_key)
return results
def increment_counter(datastore, counter_id, retries):
txid = yield datastore.begin_transaction()
key = Key.from_path('Counter', counter_id, _app=PROJECT_ID)
entity = yield datastore.get(key, txid=txid)
if entity is None:
entity = Entity('Counter', name=counter_id, _app=PROJECT_ID)
if 'count' not in entity:
entity['count'] = 0
entity['count'] += 1
yield datastore.put(entity, txid=txid)
try:
yield datastore.commit(txid)
except DatastoreError:
if retries < 1:
raise
yield increment_counter(datastore, counter_id, retries - 1)
def test_separator_in_name(locations):
datastore = Datastore(locations, PROJECT_ID)
entity = Entity('Greeting', name='Test:1', _app=PROJECT_ID)
create_time = datetime.datetime.utcnow()
entity['color'] = 'red'
entity['create_time'] = create_time
yield datastore.put(entity)
query = Query('Greeting', {
'color =': 'red',
'create_time =': create_time
},
_app=PROJECT_ID)
response = yield datastore.run_query(query)
if len(response) != 1:
raise Exception('Expected 1 result. Received: {}'.format(response))
entity = response[0]
if entity['color'] != 'red' or entity['create_time'] != create_time:
raise Exception('Unexpected entity: {}'.format(entity))
def test_merge_query_with_null(locations):
datastore = Datastore(locations, PROJECT_ID)
query = Query('Greeting', _app=PROJECT_ID)
results = yield datastore.run_query(query)
for entity in results:
yield datastore.delete([entity.key()])
entity = Entity('Greeting', _app=PROJECT_ID)
create_time = datetime.datetime.now()
entity['content'] = None
entity['create_time'] = create_time
yield datastore.put(entity)
entity = Entity('Greeting', _app=PROJECT_ID)
entity['content'] = 'hi'
entity['create_time'] = create_time
yield datastore.put(entity)
entity = Entity('Greeting', _app=PROJECT_ID)
entity['create_time'] = None
yield datastore.put(entity)
query = Query('Greeting', {
'content =': None,
'create_time =': create_time
},
_app=PROJECT_ID)
response = yield datastore.run_query(query)
if len(response) != 1:
raise Exception('Expected 1 result. Received: {}'.format(response))
entity = response[0]
if entity['content'] is not None or entity['create_time'] != create_time:
raise Exception('Unexpected entity: {}'.format(entity))
query = Query('Greeting', _app=PROJECT_ID)
results = yield datastore.run_query(query)
for entity in results:
yield datastore.delete([entity.key()])
def prepare_entity(instance):
if isinstance(instance.pk, AncestorKey):
parent = instance.pk._parent_key
else:
parent = None
result = Entity(instance._meta.db_table, parent=parent)
for field in instance._meta.fields:
if field.name == "id":
continue
value = field.pre_save(instance, True)
setattr(instance, field.name, value)
value = field.get_db_prep_save(getattr(instance, field.attname),
connection)
if isinstance(value, (list, set)):
value = list(value)
if not value:
value = None
result[field.column] = value
return result
def prep_value_for_database(self, value, index, model, column):
if value is None:
return None
# If this a date or a datetime, or something that supports isoformat, then use that
if hasattr(value, "isoformat"):
value = value.isoformat()
if _is_iterable(value):
value = list(
chain(*[self._generate_permutations(v) for v in value]))
else:
value = self._generate_permutations(value)
if not value:
return None
value = list(set(value)) # De-duplicate
entity = Entity(self._generate_kind_name(model, column),
name=self.OPERATOR)
entity[self.INDEXED_COLUMN_NAME] = value
return [entity]
def process_query_result(result):
if result.keys_only():
return [Key._FromPb(e.key()) for e in result.result_list()]
else:
return [Entity._FromPb(e) for e in result.result_list()]
logging.debug("Exception (Delete):%s", exp)
exception.append(exp)
if callback:
callback(rpc)
def get_rpc_handler(rpc):
try:
rpc.check_success()
except apiproxy_errors.ApplicationError, err:
raise _ToDatastoreError(err)
resp = rpc.response
entities = []
for group in resp.entity_list():
if group.has_entity():
entities.append(Entity._FromPb(group.entity()))
else:
entities.append(None)
return entities
def get_callback(rpc, result, exception, callback=None):
try:
assert isinstance(rpc.request, datastore_pb.GetRequest), "request should be a GetRequest"
assert isinstance(rpc.response, datastore_pb.GetResponse), "response should be a GetResponse"
response = get_rpc_handler(rpc)
result.append(response)
except (datastore_errors.Error, apiproxy_errors.Error), exp:
logging.debug("Exception (Get):%s", exp)
exception.append(exp)
def get(self):
player_entity = Entity('mykind', name = 'e_name')
player_entity['var'] = 1192
datastore.Put(player_entity)
path = os.path.join(os.path.dirname(__file__), 'view/index.html')
self.response.out.write(template.render(path, {}))
