def testDatastoreTypes(self):
"""Puts and gets different basic datastore types."""
entity = datastore.Entity('TestKind')
entity.update({
'rating':
datastore_types.Rating(1),
'category':
datastore_types.Category('bugs'),
'key':
datastore_types.Key.from_path('foo', 'bar'),
'user':
users.User('*****@*****.**'),
'text':
datastore_types.Text('some text'),
'blob':
datastore_types.Blob('data'),
'bytestring':
datastore_types.ByteString('data'),
'im':
datastore_types.IM('http://example.com/', 'Larry97'),
'geopt':
datastore_types.GeoPt(1.1234, -1.1234),
'email':
datastore_types.Email('*****@*****.**'),
'blobkey':
datastore_types.BlobKey('27f5a7'),
})
datastore.Put(entity)
e = datastore.Get(entity)
datastore.Delete(entity)
def __create_value_for_mongo_value(self, mongo_value):
if isinstance(mongo_value, Binary):
return datastore_types.Blob(str(mongo_value))
if isinstance(mongo_value, types.DictType):
if mongo_value['class'] == 'rating':
return datastore_types.Rating(int(mongo_value["rating"]))
if mongo_value['class'] == 'category':
return datastore_types.Category(mongo_value["category"])
if mongo_value['class'] == 'key':
return self.__key_for_id(mongo_value['path'])
if mongo_value['class'] == 'list':
return [
self.__create_value_for_mongo_value(v)
for v in mongo_value['list']
]
if mongo_value['class'] == 'user':
return users.User(email=mongo_value["email"])
if mongo_value['class'] == 'text':
return datastore_types.Text(mongo_value['string'])
if mongo_value['class'] == 'im':
return datastore_types.IM(mongo_value['protocol'],
mongo_value['address'])
if mongo_value['class'] == 'geopt':
return datastore_types.GeoPt(mongo_value['lat'],
mongo_value['lon'])
if mongo_value['class'] == 'email':
return datastore_types.Email(mongo_value['value'])
if mongo_value['class'] == 'bytes':
return datastore_types.ByteString(mongo_value['value'])
if mongo_value['class'] == 'blobkey':
return datastore_types.BlobKey(mongo_value['value'])
return mongo_value
def rocket_to_ae(field_type, rocket_value):
if not rocket_value:
ae_value = None
elif field_type == TYPE_DATETIME or field_type == TYPE_TIMESTAMP:
ae_value = from_iso(rocket_value)
elif field_type == TYPE_BOOL:
ae_value = bool(int(rocket_value))
elif field_type == TYPE_LONG:
ae_value = long(rocket_value)
elif field_type == TYPE_FLOAT:
ae_value = float(rocket_value)
elif field_type == TYPE_INT:
ae_value = int(rocket_value)
elif field_type == TYPE_TEXT:
ae_value = datastore_types.Text(rocket_value.replace('|', '|'))
elif field_type == TYPE_REFERENCE:
slash = rocket_value.find("/")
if slash > 0:
kind = rocket_value[:slash]
key_name_or_id = rocket_value[slash + 1:]
if key_name_or_id[0] in "0123456789":
key_name_or_id = int(key_name_or_id)
ae_value = datastore.Key.from_path(kind, key_name_or_id)
else:
logging.error("invalid reference value: %s" % rocket_value)
ae_value = None
elif field_type == TYPE_BLOB:
ae_value = datastore_types.Blob(base64.b64decode(rocket_value))
else: #str
ae_value = (u"%s" % rocket_value).replace('|', '|')
return ae_value
def transform_function(value, bulkload_state):
directory = os.path.dirname(bulkload_state.filename)
suffix = ''
filename = ''
filename_hint = value
filename = os.path.join(directory, filename_hint)
f = open(filename, 'r')
data = f.read()
f.close()
return datastore_types.Blob(data)
def save_queues_to_db(self):
""" Stores file queue information into the datastore.
Raises:
ValueError: If queue info has not been set.
"""
if not self._queue_info_file:
raise ValueError("Queue info must be set before saving the queues")
json_queues = json.dumps(self._queue_info_file)
entity = datastore.Entity(self.QUEUE_KIND,
name=self._app_id,
_app=self.APPSCALE_QUEUES)
entity.update({self.QUEUE_INFO: datastore_types.Blob(json_queues),
self.APP_NAME: datastore_types.ByteString(self._app_id)})
datastore.Put(entity)
def StoreBlob(self, blob_key, blob_stream):
"""Store blob stream to the datastore.
Args:
blob_key: Blob key of blob to store.
blob_stream: Stream or stream-like object that will generate blob content.
"""
block_count = 0
blob_key_object = self._BlobKey(blob_key)
while True:
block = blob_stream.read(blobstore.MAX_BLOB_FETCH_SIZE)
if not block:
break
entity = datastore.Entity(_BLOB_CHUNK_KIND_,
name=str(blob_key_object) + "__" +
str(block_count),
namespace='')
entity.update({'block': datastore_types.Blob(block)})
datastore.Put(entity)
block_count += 1
def js_to_appengine_value(value, attr_config):
if not value:
return None
if not attr_config:
return value
type = attr_config['type']
if type == DATETIME:
return datetime_from_iso(value)
elif type == INT:
return int(value)
elif type == LONG:
return long(value)
elif type == BOOL:
return bool(value)
elif type == TEXT:
return datastore_types.Text(value)
elif type == KEY:
kind = attr_config['kind']
if value[0] in "0123456789":
return datastore.Key.from_path(kind, int(value))
else:
return datastore.Key.from_path(kind, value)
elif type == BLOB:
return datastore_types.Blob(base64.b64decode(value))
elif type == LIST:
return map(lambda value: js_to_appengine_value(value, attr_config['items']), value.split("|"))
else: #str
return value
def StoreBlob(self, blob_key, blob_stream):
"""Store blob stream to disk.
Args:
blob_key: Blob key of blob to store.
blob_stream: Stream or stream-like object that will generate blob content.
"""
blob_key = self._BlobKey(blob_key)
block_count = 0
try:
while True:
block = blob_stream.read(blobstore.MAX_BLOB_FETCH_SIZE)
if not block:
break
entity = datastore.Entity(_BLOB_CHUNK_KIND_,
name=str(blob_key) + "__" +
str(block_count),
namespace='')
entity.update({'block': datastore_types.Blob(block)})
datastore.Put(entity)
block_count += 1
except datastore_errors.EntityNotFoundError, err:
raise apiproxy_errors.ApplicationError(
blobstore_service_pb.BlobstoreServiceError.BLOB_NOT_FOUND)
def make_value_from_form(self, value):
"""Convert a form value to a property value.
This extracts the content from the UploadedFile instance returned
by the FileField instance.
"""
if isinstance(self, ndb.StringProperty):
if not self._repeated:
return value
if not value:
return []
if isinstance(value, basestring):
logging.error("In repeated string")
value = value.splitlines()
return value
elif isinstance(self, ndb.TextProperty):
return value
else:
if isinstance(value, uploadedfile.UploadedFile):
if not self.form_value:
self.form_value = value.read()
b = datastore_types.Blob(self.form_value)
return b
return super(NDBBlobProperty, self).make_value_from_form(value)
def blobproperty_from_base64(value): """Return a datastore blob property containing the base64 decoded value.""" decoded_value = base64.b64decode(value) return datastore_types.Blob(decoded_value)
def encode(x):
value = x.encode('utf-8')
return datastore_types.Blob(value)
