def testWriteToBlobstore(self):
# There are stronger tests for the behavior of this in files_test.
old_blob_key = utils.write_to_blobstore('Blobstore!')
self.assertTrue(old_blob_key)
old_blobinfo = blobstore.BlobInfo.get(old_blob_key)
new_blob_key = utils.write_to_blobstore('Blobstore!',
old_blobinfo=old_blobinfo)
self.assertEqual(new_blob_key, old_blob_key)
self.stubs.SmartUnsetAll()
def testWriteToBlobstore(self):
# There are stronger tests for the behavior of this in files_test.
old_blob_key = utils.write_to_blobstore('Blobstore!')
self.assertTrue(old_blob_key)
old_blobinfo = blobstore.BlobInfo.get(old_blob_key)
new_blob_key = utils.write_to_blobstore('Blobstore!',
old_blobinfo=old_blobinfo)
self.assertEqual(new_blob_key, old_blob_key)
self.stubs.SmartUnsetAll()
def _maybe_write_to_blobstore(self, content, blob, force_blobstore=False):
if content and blob:
raise TypeError('Exactly one of "content" or "blob" must be given.')
if force_blobstore or content and len(content) > MAX_CONTENT_SIZE:
if not force_blobstore:
logging.debug(
'Content size %s exceeds %s bytes, uploading to blobstore.',
len(content), MAX_CONTENT_SIZE)
old_blobinfo = self.blob if self.exists else None
blob = utils.write_to_blobstore(content, old_blobinfo=old_blobinfo)
_store_blob_cache(self.real_path, content)
content = None
return content, blob
def testWrite(self):
expected_file = files._TitanFile(
id='/foo/bar.html',
name='bar.html',
content='Test',
dir_path='/foo',
paths=[u'/', u'/foo'],
depth=1,
mime_type=u'text/html',
created_by=users.TitanUser('*****@*****.**'),
modified_by=users.TitanUser('*****@*****.**'),
# Arbitrary meta data for expando:
color=u'blue',
flag=False,
md5_hash=hashlib.md5('Test').hexdigest(),
)
original_expected_file = copy.deepcopy(expected_file)
meta = {'color': 'blue', 'flag': False}
new_meta = {'color': 'blue', 'flag': True}
dates = ['modified', 'created']
# Synchronous write of a new file.
actual_file = files.File('/foo/bar.html').write('Test', meta=meta)
self.assertNdbEntityEqual(expected_file,
actual_file._file,
ignore=dates)
self.assertNotEqual(None, actual_file.modified,
'modified is not being set')
# Synchronous update without changes.
actual_file = files.File('/foo/bar.html').write(meta=meta)
self.assertNdbEntityEqual(expected_file,
actual_file._file,
ignore=dates)
# Synchronous update with changes.
old_modified = actual_file.modified
actual_file = files.File('/foo/bar.html')
actual_file.write('New content', meta=new_meta, mime_type='fake/type')
expected_file.content = 'New content'
expected_file.md5_hash = hashlib.md5('New content').hexdigest()
expected_file.flag = True
expected_file.mime_type = 'fake/type'
self.assertNdbEntityEqual(expected_file,
actual_file._file,
ignore=dates)
self.assertNotEqual(old_modified, actual_file.modified)
# Allow writing blank files.
actual_file = files.File('/foo/bar.html').write('')
self.assertEqual(actual_file.content, '')
# Allow overwriting mime_type and meta without touching content.
files.File('/foo/bar.html').write(content='Test')
actual_file = files.File('/foo/bar.html').write(
mime_type='fake/mimetype')
self.assertEqual('fake/mimetype', actual_file.mime_type)
self.assertEqual('Test', actual_file.content)
actual_file = files.File('/foo/bar.html').write(meta=new_meta)
self.assertEqual(True, actual_file.meta.flag)
self.assertEqual('Test', actual_file.content)
# Allow overwriting created and modified without touching content.
files.File('/foo/bar.html').write(content='Test')
now = datetime.datetime.now() + datetime.timedelta(days=1)
actual_file = files.File('/foo/bar.html').write(created=now,
modified=now)
self.assertEqual(now, actual_file.created)
self.assertEqual(now, actual_file.modified)
# Verify the same behavior for the file creation codepath.
files.File('/foo/bar.html').delete().write('Test',
created=now,
modified=now)
self.assertEqual(now, actual_file.created)
self.assertEqual(now, actual_file.modified)
# Error handling.
self.assertRaises(ValueError,
files.File('/a').write,
'',
created='foo')
self.assertRaises(ValueError,
files.File('/a').write,
'',
modified='foo')
# Allow overwriting created_by and modified_by without touching content.
files.File('/foo/bar.html').write(content='Test')
user = users.TitanUser(
'*****@*****.**') # Not the current logged in user.
actual_file = files.File('/foo/bar.html').write(created_by=user,
modified_by=user)
self.assertEqual(user, actual_file.created_by)
self.assertEqual(user, actual_file.modified_by)
# Verify the same behavior for the file creation codepath.
files.File('/foo/bar.html').delete().write('Test',
created_by=user,
modified_by=user)
self.assertEqual(user, actual_file.created_by)
self.assertEqual(user, actual_file.modified_by)
# Error handling.
self.assertRaises(ValueError,
files.File('/a').write,
'',
created_by='foo')
self.assertRaises(ValueError,
files.File('/a').write,
'',
modified_by='foo')
# Cleanup.
expected_file = original_expected_file
files.File('/foo/bar.html').delete()
# write large content to blobstore.
titan_file = files.File('/foo/bar.html').write(
content=LARGE_FILE_CONTENT)
blob_key = titan_file.blob.key()
self.assertTrue(blob_key)
self.assertEqual(LARGE_FILE_CONTENT, titan_file.content)
self.assertIsNone(titan_file._file_ent.content)
self.assertEqual(LARGE_FILE_CONTENT,
files.File('/foo/bar.html').content)
self.assertEqual(
hashlib.md5(LARGE_FILE_CONTENT).hexdigest(), titan_file.md5_hash)
# De-duping check: verify the blob key doesn't change if the content
# doesn't change.
old_blob_key = blob_key
titan_file = files.File('/foo/bar.html').write(
content=LARGE_FILE_CONTENT)
blob_key = titan_file.blob.key()
self.assertEqual(old_blob_key, blob_key)
self.assertEqual(LARGE_FILE_CONTENT, titan_file.content)
self.assertIsNone(titan_file._file_ent.content)
self.assertEqual(LARGE_FILE_CONTENT,
files.File('/foo/bar.html').content)
self.stubs.SmartUnsetAll()
# write with a blob key and encoding; verify proper decoding.
encoded_foo = u'f♥♥'.encode('utf-8')
blob_key = utils.write_to_blobstore(encoded_foo)
titan_file = files.File('/foo/bar.html')
# Verify that without encoding, the encoded bytestring is returned.
titan_file.write(blob=blob_key)
self.assertEqual(encoded_foo, titan_file.content)
# Verify that with encoding, a unicode string is returned.
titan_file.write(blob=blob_key, encoding='utf-8')
self.assertEqual(u'f♥♥', titan_file.content)
# Argument error handling for mixing encoding and unicode content:
self.assertRaises(TypeError,
titan_file.write,
content=u'Test',
encoding='utf-8')
# Make sure the blob is deleted with the file:
titan_file.delete()
self.assertIsNone(blobstore.get(blob_key))
self.assertRaises(files.BadFileError, lambda: titan_file.blob)
# Make sure the blob is deleted if the file gets smaller:
titan_file = files.File('/foo/bar.html').write(
content=LARGE_FILE_CONTENT)
blob_key = titan_file.blob.key()
titan_file.write(content='Test')
self.assertIsNone(blobstore.get(blob_key))
# Test the current object and a new instance:
self.assertEqual('Test', titan_file.content)
self.assertEqual('Test', files.File('/foo/bar.html').content)
self.assertIsNone(titan_file.blob)
self.assertIsNone(files.File('/foo/bar.html').blob)
# write with a BlobKey:
titan_file = files.File('/foo/bar.html').write(blob=self.blob_key)
blob_content = self.blob_reader.read()
# Test the current object and a new instance:
self.assertEqual(blob_content, files.File('/foo/bar.html').content)
self.assertEqual(blob_content, titan_file.content)
self.assertEqual(
hashlib.md5('Blobstore!').hexdigest(), titan_file.md5_hash)
# Cleanup.
expected_file = original_expected_file
files.File('/foo/bar.html').delete()
# Error handling:
# Updating mime_type or meta when entity doesn't exist.
titan_file = files.File('/fake/file')
self.assertRaises(files.BadFileError, titan_file.write, meta=meta)
self.assertRaises(files.BadFileError,
titan_file.write,
mime_type='fake/mimetype')
# Bad arguments:
self.assertRaises(TypeError, titan_file.write)
self.assertRaises(TypeError, titan_file.write, content=None, blob=None)
self.assertRaises(TypeError,
titan_file.write,
content='Test',
blob=self.blob_key)
self.assertRaises(TypeError, titan_file.write, encoding='utf-8')
# There are some reserved words that cannot be used in meta properties.
invalid_meta_keys = [
# Titan reserved:
'name',
'path',
'dir_path',
'paths',
'depth',
'mime_type',
'encoding',
'created',
'modified',
'content',
'blob',
'blobs',
'created_by',
'modified_by',
'md5_hash',
# NDB reserved:
'key',
'app',
'id',
'parent',
'namespace',
'projection',
]
for key in invalid_meta_keys:
try:
titan_file.write(content='', meta={key: ''})
except files.InvalidMetaError:
pass
else:
self.fail(
'Invalid meta key should have failed: {!r}'.format(key))
def testWrite(self):
expected_file = files._TitanFile(
id='/foo/bar.html',
name='bar.html',
content='Test',
dir_path='/foo',
paths=[u'/', u'/foo'],
depth=1,
mime_type=u'text/html',
created_by=users.TitanUser('*****@*****.**'),
modified_by=users.TitanUser('*****@*****.**'),
# Arbitrary meta data for expando:
color=u'blue',
flag=False,
md5_hash=hashlib.md5('Test').hexdigest(),
)
original_expected_file = copy.deepcopy(expected_file)
meta = {'color': 'blue', 'flag': False}
new_meta = {'color': 'blue', 'flag': True}
dates = ['modified', 'created']
# Synchronous write of a new file.
actual_file = files.File('/foo/bar.html').write('Test', meta=meta)
self.assertNdbEntityEqual(expected_file, actual_file._file, ignore=dates)
self.assertNotEqual(None, actual_file.modified, 'modified is not being set')
# Synchronous update without changes.
actual_file = files.File('/foo/bar.html').write(meta=meta)
self.assertNdbEntityEqual(expected_file, actual_file._file, ignore=dates)
# Synchronous update with changes.
old_modified = actual_file.modified
actual_file = files.File('/foo/bar.html')
actual_file.write('New content', meta=new_meta, mime_type='fake/type')
expected_file.content = 'New content'
expected_file.md5_hash = hashlib.md5('New content').hexdigest()
expected_file.flag = True
expected_file.mime_type = 'fake/type'
self.assertNdbEntityEqual(expected_file, actual_file._file, ignore=dates)
self.assertNotEqual(old_modified, actual_file.modified)
# Allow writing blank files.
actual_file = files.File('/foo/bar.html').write('')
self.assertEqual(actual_file.content, '')
# Allow overwriting mime_type and meta without touching content.
files.File('/foo/bar.html').write(content='Test')
actual_file = files.File('/foo/bar.html').write(mime_type='fake/mimetype')
self.assertEqual('fake/mimetype', actual_file.mime_type)
self.assertEqual('Test', actual_file.content)
actual_file = files.File('/foo/bar.html').write(meta=new_meta)
self.assertEqual(True, actual_file.meta.flag)
self.assertEqual('Test', actual_file.content)
# Allow overwriting created and modified without touching content.
files.File('/foo/bar.html').write(content='Test')
now = datetime.datetime.now() + datetime.timedelta(days=1)
actual_file = files.File('/foo/bar.html').write(created=now, modified=now)
self.assertEqual(now, actual_file.created)
self.assertEqual(now, actual_file.modified)
# Verify the same behavior for the file creation codepath.
files.File('/foo/bar.html').delete().write(
'Test', created=now, modified=now)
self.assertEqual(now, actual_file.created)
self.assertEqual(now, actual_file.modified)
# Error handling.
self.assertRaises(ValueError, files.File('/a').write, '', created='foo')
self.assertRaises(ValueError, files.File('/a').write, '', modified='foo')
# Allow overwriting created_by and modified_by without touching content.
files.File('/foo/bar.html').write(content='Test')
user = users.TitanUser('*****@*****.**') # Not the current logged in user.
actual_file = files.File('/foo/bar.html').write(
created_by=user, modified_by=user)
self.assertEqual(user, actual_file.created_by)
self.assertEqual(user, actual_file.modified_by)
# Verify the same behavior for the file creation codepath.
files.File('/foo/bar.html').delete().write(
'Test', created_by=user, modified_by=user)
self.assertEqual(user, actual_file.created_by)
self.assertEqual(user, actual_file.modified_by)
# Error handling.
self.assertRaises(ValueError, files.File('/a').write, '', created_by='foo')
self.assertRaises(ValueError, files.File('/a').write, '', modified_by='foo')
# Cleanup.
expected_file = original_expected_file
files.File('/foo/bar.html').delete()
# write large content to blobstore.
titan_file = files.File('/foo/bar.html').write(content=LARGE_FILE_CONTENT)
blob_key = titan_file.blob.key()
self.assertTrue(blob_key)
self.assertEqual(LARGE_FILE_CONTENT, titan_file.content)
self.assertIsNone(titan_file._file_ent.content)
self.assertEqual(LARGE_FILE_CONTENT, files.File('/foo/bar.html').content)
self.assertEqual(hashlib.md5(LARGE_FILE_CONTENT).hexdigest(),
titan_file.md5_hash)
# De-duping check: verify the blob key doesn't change if the content
# doesn't change.
old_blob_key = blob_key
titan_file = files.File('/foo/bar.html').write(content=LARGE_FILE_CONTENT)
blob_key = titan_file.blob.key()
self.assertEqual(old_blob_key, blob_key)
self.assertEqual(LARGE_FILE_CONTENT, titan_file.content)
self.assertIsNone(titan_file._file_ent.content)
self.assertEqual(LARGE_FILE_CONTENT, files.File('/foo/bar.html').content)
self.stubs.SmartUnsetAll()
# write with a blob key and encoding; verify proper decoding.
encoded_foo = u'f♥♥'.encode('utf-8')
blob_key = utils.write_to_blobstore(encoded_foo)
titan_file = files.File('/foo/bar.html')
# Verify that without encoding, the encoded bytestring is returned.
titan_file.write(blob=blob_key)
self.assertEqual(encoded_foo, titan_file.content)
# Verify that with encoding, a unicode string is returned.
titan_file.write(blob=blob_key, encoding='utf-8')
self.assertEqual(u'f♥♥', titan_file.content)
# Argument error handling for mixing encoding and unicode content:
self.assertRaises(TypeError, titan_file.write, content=u'Test',
encoding='utf-8')
# Make sure the blob is deleted with the file:
titan_file.delete()
self.assertIsNone(blobstore.get(blob_key))
self.assertRaises(files.BadFileError, lambda: titan_file.blob)
# Make sure the blob is deleted if the file gets smaller:
titan_file = files.File('/foo/bar.html').write(content=LARGE_FILE_CONTENT)
blob_key = titan_file.blob.key()
titan_file.write(content='Test')
self.assertIsNone(blobstore.get(blob_key))
# Test the current object and a new instance:
self.assertEqual('Test', titan_file.content)
self.assertEqual('Test', files.File('/foo/bar.html').content)
self.assertIsNone(titan_file.blob)
self.assertIsNone(files.File('/foo/bar.html').blob)
# write with a BlobKey:
titan_file = files.File('/foo/bar.html').write(blob=self.blob_key)
blob_content = self.blob_reader.read()
# Test the current object and a new instance:
self.assertEqual(blob_content, files.File('/foo/bar.html').content)
self.assertEqual(blob_content, titan_file.content)
self.assertEqual(hashlib.md5('Blobstore!').hexdigest(),
titan_file.md5_hash)
# Cleanup.
expected_file = original_expected_file
files.File('/foo/bar.html').delete()
# Error handling:
# Updating mime_type or meta when entity doesn't exist.
titan_file = files.File('/fake/file')
self.assertRaises(files.BadFileError, titan_file.write, meta=meta)
self.assertRaises(files.BadFileError, titan_file.write,
mime_type='fake/mimetype')
# Bad arguments:
self.assertRaises(TypeError, titan_file.write)
self.assertRaises(TypeError, titan_file.write, content=None, blob=None)
self.assertRaises(TypeError, titan_file.write, content='Test',
blob=self.blob_key)
self.assertRaises(TypeError, titan_file.write, encoding='utf-8')
# There are some reserved words that cannot be used in meta properties.
invalid_meta_keys = [
# Titan reserved:
'name',
'path',
'dir_path',
'paths',
'depth',
'mime_type',
'encoding',
'created',
'modified',
'content',
'blob',
'blobs',
'created_by',
'modified_by',
'md5_hash',
# NDB reserved:
'key',
'app',
'id',
'parent',
'namespace',
'projection',
]
for key in invalid_meta_keys:
try:
titan_file.write(content='', meta={key: ''})
except files.InvalidMetaError:
pass
else:
self.fail(
'Invalid meta key should have failed: {!r}'.format(key))
